2023-07-14T10:37:30.141 INFO:root:teuthology version: 0.0.1.dev145+gaa6a80e 2023-07-14T10:37:30.142 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_rishabh@teuthology --name rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi --verbose --archive /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403 --description fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{no-subvolume} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} -- /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/orig.config.yaml 2023-07-14T10:37:30.203 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-07-14T10:37:30.298 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403 branch: wip-rishabh-2023Jul13 description: fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{no-subvolume} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} email: null first_in_suite: false job_id: '7337403' kernel: client: branch: testing flavor: default kdb: true sha1: a440c79eee0258ae07e4988d8690fe6187b8b7a3 ktype: upstream last_in_suite: false log-rotate: ceph-mds: 10G ceph-osd: 10G machine_type: smithi name: rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 20 - machine: disk: 200 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: wip-rishabh-2023Jul13 ceph: cephfs: ec_profile: - m=2 - k=2 - crush-failure-domain=osd max_mds: 5 session_timeout: 300 standby_replay: true conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: ms die on skipped message: false mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds bal fragment size max: 10000 mds bal merge size: 5 mds bal split bits: 3 mds bal split size: 100 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: cephsqlite lock renewal timeout: 900000 debug cephsqlite: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug ms: 1 debug osd: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 osd shutdown pgref assert: true osd_max_omap_entries_per_request: 10000 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - overall HEALTH_ - \(FS_DEGRADED\) - \(MDS_FAILED\) - \(MDS_DEGRADED\) - \(FS_WITH_FAILED_MDS\) - \(MDS_DAMAGE\) - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(FS_INLINE_DATA_DEPRECATED\) - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - slow metadata IO - SLOW_OPS - slow request sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: osd default pool size: 2 osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 20 debug bluestore: 20 debug rocksdb: 10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs ceph-fuse: client.0: mntopts: - context=system_u:object_r:postgresql_db_t:s0 mountpoint: /tmp/cephfs check-counter: counters: mds: - mds.exported - mds.imported - mds.root_rsnaps - mds_server.req_mksnap_latency.avgcount - mds_server.req_rmsnap_latency.avgcount install: ceph: flavor: default sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b kclient: client.0: mntopts: - context=system_u:object_r:postgresql_db_t:s0 mountpoint: /tmp/cephfs mntopts: - ms_mode=crc - wsync syntax: v1 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-rishabh-2023Jul13 sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b owner: scheduled_rishabh@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mgr.x - mds.a - mds.d - mds.g - mds.j - osd.0 - osd.3 - osd.6 - osd.9 - client.0 - - mon.b - mgr.y - mds.b - mds.e - mds.h - mds.k - osd.1 - osd.4 - osd.7 - osd.10 - - mon.c - mgr.z - mds.c - mds.f - mds.i - mds.l - osd.2 - osd.5 - osd.8 - osd.11 sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b sleep_before_teardown: 0 suite: fs suite_branch: wip-rishabh-2023Jul13 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b syslog: ignorelist: - WARNING*.*check_session_state - WARNING*.*__ceph_remove_cap targets: smithi040.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= smithi045.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= smithi130.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: extra_packages: deb: - python3-cephfs - cephfs-shell - cephfs-top - cephfs-mirror rpm: - python3-cephfs - cephfs-top - cephfs-mirror extra_system_packages: deb: - bison - flex - libelf-dev - libssl-dev - network-manager - iproute2 - util-linux - dump - indent - libaio-dev - libtool-bin - uuid-dev - xfslibs-dev - postgresql - postgresql-client - postgresql-common - postgresql-contrib rpm: - bison - flex - elfutils-libelf-devel - openssl-devel - NetworkManager - iproute - util-linux - libacl-devel - libaio-devel - libattr-devel - libtool - libuuid-devel - xfsdump - xfsprogs - xfsprogs-devel - libaio-devel - libtool - libuuid-devel - xfsprogs-devel - postgresql - postgresql-server - postgresql-contrib - cephadm: roleless: false - cephadm.shell: mon.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: mon.a: - ceph fs dump - ceph osd dump - fs.ready: timeout: 300 - kclient: null - check-counter: {} - fwd_scrub: scrub_timeout: 900 sleep_between_iterations: 1 - exec: mon.a: - ceph mgr module enable snap_schedule - ceph config set mgr mgr/snap_schedule/allow_m_granularity true - ceph config set mgr mgr/snap_schedule/dump_on_update true - ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1M - ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6M3h - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - date +%s > START_TIME - full_sequential_finally: - exec: mon.a: - date +%s > END_TIME - START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi - ceph fs snap-schedule status --fs=cephfs --path=/ - ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true - exec: client.0: - sudo ls -l /tmp/cephfs/ && sudo df -h /tmp/cephfs/ - sudo mkdir -m 755 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres && sudo chown postgres:postgres /tmp/cephfs/postgres - sudo -u postgres -- mkdir -m 700 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres/data - sudo semanage fcontext -a -t postgresql_db_t "/tmp/cephfs/postgres(/.*)?" - sudo grep -i postgresql /etc/selinux/targeted/contexts/files/file_contexts.local - sudo restorecon -R -v /tmp/cephfs/postgres - sudo ls -lZaR /tmp/cephfs/postgres/ - sudo mkdir -p /etc/systemd/system/postgresql.service.d/ && printf '[Service]\nEnvironment=PGDATA=/tmp/cephfs/postgres/data\nEnvironment=PGLOG=/tmp/cephfs/postgres/pgstartup.log\n' | sudo tee /etc/systemd/system/postgresql.service.d/env.conf - sudo -u postgres -- postgresql-setup --initdb - sudo ls -lZaR /tmp/cephfs/postgres/ - sudo systemctl start postgresql - sudo -u postgres -- pgbench -s 500 -i - sudo -u postgres -- pgbench -c 100 -j 4 --progress=5 --time=900 - sudo systemctl stop postgresql - sudo ls -lZaR /tmp/cephfs/postgres/ teuthology: fragments_dropped: [] meta: {} postmerge: - local function is_kupstream() return yaml.ktype == 'upstream' end - local function is_kdistro() return yaml.ktype == 'distro' end - local function is_fuse() return false end - local function syntax_version() return yaml.overrides.kclient.syntax end teuthology_branch: main teuthology_sha1: aa6a80e6cc655fff48452a7eca02070e61d074fc timestamp: 2023-07-14_10:26:42 tube: smithi user: rishabh verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.23459 2023-07-14T10:37:30.299 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa; will attempt to use it 2023-07-14T10:37:30.299 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks 2023-07-14T10:37:30.300 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-07-14T10:37:30.301 INFO:teuthology.task.internal:Checking packages... 2023-07-14T10:37:30.329 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash '3fc0089cc1082313a98853d48b892acdc1385b8b' 2023-07-14T10:37:30.329 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-07-14T10:37:30.329 INFO:teuthology.packaging:ref: None 2023-07-14T10:37:30.330 INFO:teuthology.packaging:tag: None 2023-07-14T10:37:30.330 INFO:teuthology.packaging:branch: wip-rishabh-2023Jul13 2023-07-14T10:37:30.330 INFO:teuthology.packaging:sha1: 3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T10:37:30.330 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-rishabh-2023Jul13 2023-07-14T10:37:30.576 INFO:teuthology.task.internal:Found packages for ceph version 18.0.0-4757.g3fc0089c 2023-07-14T10:37:30.577 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-07-14T10:37:30.583 INFO:teuthology.task.internal:no buildpackages task found 2023-07-14T10:37:30.584 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-07-14T10:37:30.607 INFO:teuthology.task.internal:Saving configuration 2023-07-14T10:37:30.629 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-07-14T10:37:30.640 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-07-14T10:37:30.669 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.462832', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-07-14T10:37:30.689 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi045.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.466833', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-07-14T10:37:30.711 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.464971', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-07-14T10:37:30.711 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-07-14T10:37:30.719 INFO:teuthology.task.internal:roles: ubuntu@smithi040.front.sepia.ceph.com - ['mon.a', 'mgr.x', 'mds.a', 'mds.d', 'mds.g', 'mds.j', 'osd.0', 'osd.3', 'osd.6', 'osd.9', 'client.0'] 2023-07-14T10:37:30.720 INFO:teuthology.task.internal:roles: ubuntu@smithi045.front.sepia.ceph.com - ['mon.b', 'mgr.y', 'mds.b', 'mds.e', 'mds.h', 'mds.k', 'osd.1', 'osd.4', 'osd.7', 'osd.10'] 2023-07-14T10:37:30.720 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['mon.c', 'mgr.z', 'mds.c', 'mds.f', 'mds.i', 'mds.l', 'osd.2', 'osd.5', 'osd.8', 'osd.11'] 2023-07-14T10:37:30.720 INFO:teuthology.run_tasks:Running task console_log... 2023-07-14T10:37:30.830 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9ddde7a5e0>, signals=[15]) 2023-07-14T10:37:30.831 INFO:teuthology.run_tasks:Running task internal.connect... 2023-07-14T10:37:30.842 INFO:teuthology.task.internal:Opening connections... 2023-07-14T10:37:30.843 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T10:37:30.844 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:37:30.914 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi045.front.sepia.ceph.com 2023-07-14T10:37:30.915 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:37:30.984 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2023-07-14T10:37:30.985 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:37:31.051 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-07-14T10:37:31.063 DEBUG:teuthology.orchestra.run.smithi040:> uname -m 2023-07-14T10:37:31.085 INFO:teuthology.orchestra.run.smithi040.stdout:x86_64 2023-07-14T10:37:31.085 DEBUG:teuthology.orchestra.run.smithi040:> cat /etc/os-release 2023-07-14T10:37:31.145 INFO:teuthology.orchestra.run.smithi040.stdout:NAME="Red Hat Enterprise Linux" 2023-07-14T10:37:31.145 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION="8.6 (Ootpa)" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:ID="rhel" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:ID_LIKE="fedora" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION_ID="8.6" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:PLATFORM_ID="platform:el8" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:ANSI_COLOR="0;31" 2023-07-14T10:37:31.146 INFO:teuthology.orchestra.run.smithi040.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:HOME_URL="https://www.redhat.com/" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-07-14T10:37:31.147 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-07-14T10:37:31.148 INFO:teuthology.lock.ops:Updating smithi040.front.sepia.ceph.com on lock server 2023-07-14T10:37:31.173 DEBUG:teuthology.orchestra.run.smithi045:> uname -m 2023-07-14T10:37:31.193 INFO:teuthology.orchestra.run.smithi045.stdout:x86_64 2023-07-14T10:37:31.193 DEBUG:teuthology.orchestra.run.smithi045:> cat /etc/os-release 2023-07-14T10:37:31.253 INFO:teuthology.orchestra.run.smithi045.stdout:NAME="Red Hat Enterprise Linux" 2023-07-14T10:37:31.253 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION="8.6 (Ootpa)" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:ID="rhel" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:ID_LIKE="fedora" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION_ID="8.6" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:PLATFORM_ID="platform:el8" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:ANSI_COLOR="0;31" 2023-07-14T10:37:31.254 INFO:teuthology.orchestra.run.smithi045.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:HOME_URL="https://www.redhat.com/" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-07-14T10:37:31.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-07-14T10:37:31.256 INFO:teuthology.lock.ops:Updating smithi045.front.sepia.ceph.com on lock server 2023-07-14T10:37:31.281 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2023-07-14T10:37:31.302 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2023-07-14T10:37:31.302 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2023-07-14T10:37:31.361 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="Red Hat Enterprise Linux" 2023-07-14T10:37:31.361 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="8.6 (Ootpa)" 2023-07-14T10:37:31.361 INFO:teuthology.orchestra.run.smithi130.stdout:ID="rhel" 2023-07-14T10:37:31.361 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE="fedora" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="8.6" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:PLATFORM_ID="platform:el8" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:ANSI_COLOR="0;31" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://www.redhat.com/" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-07-14T10:37:31.362 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-07-14T10:37:31.363 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:37:31.363 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-07-14T10:37:31.363 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-07-14T10:37:31.363 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-07-14T10:37:31.363 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-07-14T10:37:31.363 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2023-07-14T10:37:31.389 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-07-14T10:37:31.401 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-07-14T10:37:31.409 INFO:teuthology.task.internal:Checking for old test directory... 2023-07-14T10:37:31.410 DEBUG:teuthology.orchestra.run.smithi040:> test '!' -e /home/ubuntu/cephtest 2023-07-14T10:37:31.412 DEBUG:teuthology.orchestra.run.smithi045:> test '!' -e /home/ubuntu/cephtest 2023-07-14T10:37:31.414 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2023-07-14T10:37:31.431 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-07-14T10:37:31.440 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-07-14T10:37:31.440 DEBUG:teuthology.orchestra.run.smithi040:> test -z $(ls -A /var/lib/ceph) 2023-07-14T10:37:31.471 DEBUG:teuthology.orchestra.run.smithi045:> test -z $(ls -A /var/lib/ceph) 2023-07-14T10:37:31.473 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2023-07-14T10:37:31.509 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-07-14T10:37:31.596 INFO:teuthology.run_tasks:Running task kernel... 2023-07-14T10:37:31.618 INFO:teuthology.task.kernel:normalize config orig: {'client': {'branch': 'testing'}, 'flavor': 'default', 'kdb': True, 'sha1': 'a440c79eee0258ae07e4988d8690fe6187b8b7a3'} 2023-07-14T10:37:31.619 INFO:teuthology.task.kernel:normalize config final: {'client.0': {'branch': 'testing'}} 2023-07-14T10:37:31.619 INFO:teuthology.task.kernel:config {'client.0': {'branch': 'testing'}}, timeout 300 2023-07-14T10:37:31.619 DEBUG:teuthology.orchestra.run.smithi040:> test -f /run/.containerenv -o -f /.dockerenv 2023-07-14T10:37:31.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:37:31.639 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F8%2Fx86_64&ref=testing 2023-07-14T10:37:31.900 DEBUG:teuthology.task.kernel:sha1 for client.0 is a440c79eee0258ae07e4988d8690fe6187b8b7a3 2023-07-14T10:37:31.900 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "a440c79eee0258ae07e4988d8690fe6187b8b7a3"... 2023-07-14T10:37:31.901 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2023-07-14T10:37:31.921 INFO:teuthology.orchestra.run.smithi040.stdout:4.18.0-372.9.1.el8.x86_64 2023-07-14T10:37:31.921 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-372.9.1.el8.x86_64 vs a440c79eee0258ae07e4988d8690fe6187b8b7a3 2023-07-14T10:37:31.922 DEBUG:teuthology.task.kernel:failed to parse current kernel version 2023-07-14T10:37:31.922 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum upgrade -y linux-firmware 2023-07-14T10:37:33.169 INFO:teuthology.orchestra.run.smithi040.stdout:Updating Subscription Management repositories. 2023-07-14T10:37:33.169 INFO:teuthology.orchestra.run.smithi040.stdout:Unable to read consumer identity 2023-07-14T10:37:34.618 INFO:teuthology.orchestra.run.smithi040.stdout:Copr repo for python3-asyncssh owned by ceph 17 kB/s | 3.5 kB 00:00 2023-07-14T10:37:38.710 INFO:teuthology.orchestra.run.smithi040.stdout:Extra Packages for Enterprise Linux 3.8 MB/s | 16 MB 00:04 2023-07-14T10:37:42.285 INFO:teuthology.orchestra.run.smithi040.stdout:lab-extras 413 kB/s | 24 kB 00:00 2023-07-14T10:37:43.032 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-07-14T10:37:43.033 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-07-14T10:37:43.033 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-07-14T10:37:43.162 INFO:teuthology.task.kernel:Downloading kernel a440c79eee0258ae07e4988d8690fe6187b8b7a3 on client.0... 2023-07-14T10:37:43.163 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=kernel&flavor=default&distros=centos%2F8%2Fx86_64&sha1=a440c79eee0258ae07e4988d8690fe6187b8b7a3 2023-07-14T10:37:43.388 INFO:teuthology.task.kernel:fetching, builder baseurl is https://2.chacra.ceph.com/r/kernel/testing/a440c79eee0258ae07e4988d8690fe6187b8b7a3/centos/8/flavors/default/x86_64/ 2023-07-14T10:37:43.388 DEBUG:teuthology.orchestra.run.smithi040:> rm -f /tmp/kernel.x86_64.rpm && echo kernel-6.5.0_rc1_ga440c79eee02-1.x86_64.rpm | wget -nv -O /tmp/kernel.x86_64.rpm --base=https://2.chacra.ceph.com/r/kernel/testing/a440c79eee0258ae07e4988d8690fe6187b8b7a3/centos/8/flavors/default/x86_64/ --input-file=- 2023-07-14T10:37:43.391 DEBUG:teuthology.task.kernel:Waiting for download/copy to ubuntu@smithi040.front.sepia.ceph.com to complete... 2023-07-14T10:38:46.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14 10:38:46 URL:https://2.chacra.ceph.com/r/kernel/testing/a440c79eee0258ae07e4988d8690fe6187b8b7a3/centos/8/flavors/default/x86_64/kernel-6.5.0_rc1_ga440c79eee02-1.x86_64.rpm [380684764/380684764] -> "/tmp/kernel.x86_64.rpm" [1] 2023-07-14T10:38:46.452 INFO:teuthology.orchestra.run.smithi040.stderr:FINISHED --2023-07-14 10:38:46-- 2023-07-14T10:38:46.452 INFO:teuthology.orchestra.run.smithi040.stderr:Total wall clock time: 1m 3s 2023-07-14T10:38:46.452 INFO:teuthology.orchestra.run.smithi040.stderr:Downloaded: 1 files, 363M in 1m 3s (5.79 MB/s) 2023-07-14T10:38:46.453 INFO:teuthology.task.kernel:Installing kernel a440c79eee0258ae07e4988d8690fe6187b8b7a3 on client.0... 2023-07-14T10:38:46.454 DEBUG:teuthology.orchestra.run.smithi040:> sudo rpm -ivh --oldpackage --replacefiles --replacepkgs /tmp/kernel.x86_64.rpm 2023-07-14T10:38:48.146 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying... ######################################## 2023-07-14T10:38:48.198 INFO:teuthology.orchestra.run.smithi040.stdout:Preparing... ######################################## 2023-07-14T10:38:48.284 INFO:teuthology.orchestra.run.smithi040.stdout:Updating / installing... 2023-07-14T10:39:29.079 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-6.5.0_rc1_ga440c79eee02-1 ######################################## 2023-07-14T10:39:56.915 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi040.front.sepia.ceph.com, path=/tmp/kernel.x86_64.rpm, version=None) 2023-07-14T10:39:56.916 DEBUG:teuthology.orchestra.run.smithi040:> rpm -qlp /tmp/kernel.x86_64.rpm 2023-07-14T10:39:56.964 INFO:teuthology.orchestra.run.smithi040.stdout:/boot/System.map-6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/boot/config-6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/boot/vmlinuz-6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch 2023-07-14T10:39:56.965 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/blowfish-x86_64.ko 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/camellia-aesni-avx-x86_64.ko 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/camellia-aesni-avx2.ko 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/camellia-x86_64.ko 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/cast5-avx-x86_64.ko 2023-07-14T10:39:56.966 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/cast6-avx-x86_64.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/crc32-pclmul.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/crc32c-intel.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/crct10dif-pclmul.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/des3_ede-x86_64.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/ghash-clmulni-intel.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/serpent-avx-x86_64.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/serpent-avx2.ko 2023-07-14T10:39:56.967 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/serpent-sse2-x86_64.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/sha1-ssse3.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/sha256-ssse3.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/sha512-ssse3.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/twofish-avx-x86_64.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/twofish-x86_64-3way.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/crypto/twofish-x86_64.ko 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kernel 2023-07-14T10:39:56.968 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kernel/cpu 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kernel/cpu/mce 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kernel/cpu/mce/mce-inject.ko 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kvm 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kvm/kvm-amd.ko 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kvm/kvm-intel.ko 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/arch/x86/kvm/kvm.ko 2023-07-14T10:39:56.969 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/ansi_cprng.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/anubis.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/arc4.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/async_memcpy.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/async_pq.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/async_raid6_recov.ko 2023-07-14T10:39:56.970 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/async_tx.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/async_xor.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/async_tx/raid6test.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/authenc.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/authencesn.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/blake2b_generic.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/blowfish_common.ko 2023-07-14T10:39:56.971 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/blowfish_generic.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/camellia_generic.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/cast5_generic.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/cast6_generic.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/cast_common.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/ccm.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/cmac.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/crc32_generic.ko 2023-07-14T10:39:56.972 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/crypto_engine.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/crypto_user.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/cts.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/des_generic.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/dh_generic.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/echainiv.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/essiv.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/fcrypt.ko 2023-07-14T10:39:56.973 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/khazad.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/lz4.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/lz4hc.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/md4.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/michael_mic.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/pcbc.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/pcrypt.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/rmd160.ko 2023-07-14T10:39:56.974 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/seed.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/serpent_generic.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/tcrypt.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/tea.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/twofish_common.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/twofish_generic.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/vmac.ko 2023-07-14T10:39:56.975 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/wp512.ko 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/xcbc.ko 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/xor.ko 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/crypto/xxhash_generic.ko 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/acpi_ipmi.ko 2023-07-14T10:39:56.976 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/acpi_pad.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/custom_method.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/ec_sys.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/platform_profile.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/sbs.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/sbshc.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/acpi/video.ko 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata 2023-07-14T10:39:56.977 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/acard-ahci.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/ahci_platform.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/ata_generic.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/libahci_platform.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_acpi.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_ali.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_amd.ko 2023-07-14T10:39:56.978 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_artop.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_atiixp.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_atp867x.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_cmd640.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_cmd64x.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_cypress.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_efar.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_hpt366.ko 2023-07-14T10:39:56.979 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_hpt37x.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_hpt3x2n.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_hpt3x3.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_it8213.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_it821x.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_jmicron.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_marvell.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_mpiix.ko 2023-07-14T10:39:56.980 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_netcell.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_ninja32.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_ns87410.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_ns87415.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_oldpiix.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_opti.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_optidma.ko 2023-07-14T10:39:56.981 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_pcmcia.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_pdc2027x.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_pdc202xx_old.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_piccolo.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_rdc.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_sch.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_serverworks.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_sil680.ko 2023-07-14T10:39:56.982 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_sis.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_sl82c105.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_triflex.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pata_via.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/pdc_adma.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_inic162x.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_mv.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_nv.ko 2023-07-14T10:39:56.983 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_promise.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_qstor.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_sil.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_sil24.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_sis.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_svw.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_sx4.ko 2023-07-14T10:39:56.984 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_uli.ko 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_via.ko 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ata/sata_vsc.ko 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/auxdisplay 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/auxdisplay/ks0108.ko 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/base 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/base/regmap 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/base/regmap/regmap-i2c.ko 2023-07-14T10:39:56.985 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/base/regmap/regmap-mmio.ko 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/bcma 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/bcma/bcma.ko 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/aoe 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/aoe/aoe.ko 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/brd.ko 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/drbd 2023-07-14T10:39:56.986 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/drbd/drbd.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/floppy.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/loop.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/mtip32xx 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/mtip32xx/mtip32xx.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/nbd.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/null_blk 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/null_blk/null_blk.ko 2023-07-14T10:39:56.987 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/pktcdvd.ko 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/rbd.ko 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/virtio_blk.ko 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/xen-blkback 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/xen-blkback/xen-blkback.ko 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/xen-blkfront.ko 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/zram 2023-07-14T10:39:56.988 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/block/zram/zram.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hangcheck-timer.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random/amd-rng.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random/intel-rng.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random/timeriomem-rng.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random/via-rng.ko 2023-07-14T10:39:56.989 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/hw_random/virtio-rng.ko 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_devintf.ko 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_msghandler.ko 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_poweroff.ko 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_si.ko 2023-07-14T10:39:56.990 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_ssif.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ipmi/ipmi_watchdog.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/lp.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/mwave 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/mwave/mwave.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/ppdev.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tlclk.ko 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm 2023-07-14T10:39:56.991 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm_atmel.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm_infineon.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm_nsc.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm_tis.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/tpm/tpm_tis_core.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/uv_mmtimer.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/char/virtio_console.ko 2023-07-14T10:39:56.992 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/acpi-cpufreq.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/amd_freq_sensitivity.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/p4-clockmod.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/pcc-cpufreq.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/powernow-k8.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/cpufreq/speedstep-lib.ko 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto 2023-07-14T10:39:56.993 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/ccp 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/ccp/ccp-crypto.ko 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/ccp/ccp.ko 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel/qat 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel/qat/qat_common 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel/qat/qat_common/intel_qat.ko 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel/qat/qat_dh895xcc 2023-07-14T10:39:56.994 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/intel/qat/qat_dh895xcc/qat_dh895xcc.ko 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/padlock-aes.ko 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/padlock-sha.ko 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/virtio 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/crypto/virtio/virtio_crypto.ko 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dca 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dca/dca.ko 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/devfreq 2023-07-14T10:39:56.995 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/devfreq/governor_simpleondemand.ko 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dma 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dma/dw 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dma/dw/dw_dmac.ko 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dma/ioat 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/dma/ioat/ioatdma.ko 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac 2023-07-14T10:39:56.996 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/amd64_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/e752x_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/edac_mce_amd.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i3000_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i3200_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i5100_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i5400_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i7300_edac.ko 2023-07-14T10:39:56.997 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i7core_edac.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/i82975x_edac.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/ie31200_edac.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/sb_edac.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/edac/x38_edac.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/firmware 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/firmware/edd.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/firmware/iscsi_ibft.ko 2023-07-14T10:39:56.998 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/gpio 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/gpio/gpio-ich.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/gpio/gpio-viperboard.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-appleir.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-aureal.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-axff.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-dr.ko 2023-07-14T10:39:56.999 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-elecom.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-elo.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-emsff.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-gaff.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-gt683r.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-gyration.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-holtek-kbd.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-holtek-mouse.ko 2023-07-14T10:39:57.000 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-holtekff.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-hyperv.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-icade.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-keytouch.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-kye.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-lcpower.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-led.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-lenovo.ko 2023-07-14T10:39:57.001 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-logitech-dj.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-logitech-hidpp.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-multitouch.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-ortek.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-penmount.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-petalynx.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-picolcd.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-pl.ko 2023-07-14T10:39:57.002 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-plantronics.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-primax.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-rmi.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-arvo.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-common.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-isku.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-kone.ko 2023-07-14T10:39:57.003 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-koneplus.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-konepure.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-kovaplus.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-lua.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-pyra.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-ryos.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat-savu.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-roccat.ko 2023-07-14T10:39:57.004 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-saitek.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-samsung.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-sensor-hub.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-sjoy.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-sony.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-speedlink.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-steelseries.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-sunplus.ko 2023-07-14T10:39:57.005 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-thrustmaster.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-tivo.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-tmff.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-topseed.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-twinhan.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-uclogic.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-waltop.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-wiimote.ko 2023-07-14T10:39:57.006 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-xinmo.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-zpff.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/hid-zydacron.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/uhid.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hid/wacom.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hv 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hv/hv_balloon.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hv/hv_utils.ko 2023-07-14T10:39:57.007 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hv/hv_vmbus.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/abituguru.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/abituguru3.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/acpi_power_meter.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ad7414.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ad7418.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adc128d818.ko 2023-07-14T10:39:57.008 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adm1025.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adm1026.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adm1029.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adm1031.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adm9240.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ads7828.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7410.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7411.ko 2023-07-14T10:39:57.009 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7462.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7470.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7475.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/adt7x10.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/amc6821.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/applesmc.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/asb100.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/asc7621.ko 2023-07-14T10:39:57.010 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/asus_atk0110.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/atxp1.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/coretemp.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/dell-smm-hwmon.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/dme1737.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ds1621.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ds620.ko 2023-07-14T10:39:57.011 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/emc1403.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/emc6w201.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/f71805f.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/f71882fg.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/f75375s.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/fam15h_power.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/fschmd.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/g760a.ko 2023-07-14T10:39:57.012 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/g762.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/gl518sm.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/gl520sm.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/hwmon-vid.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/i5500_temp.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/i5k_amb.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ibmaem.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ibmpex.ko 2023-07-14T10:39:57.013 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ina209.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ina2xx.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/it87.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/k10temp.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/k8temp.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lineage-pem.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm63.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm73.ko 2023-07-14T10:39:57.014 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm75.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm77.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm78.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm80.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm83.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm85.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm87.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm90.ko 2023-07-14T10:39:57.015 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm92.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm93.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm95234.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm95241.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/lm95245.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc2945.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4151.ko 2023-07-14T10:39:57.016 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4215.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4222.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4245.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4260.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/ltc4261.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max16065.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max1619.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max1668.ko 2023-07-14T10:39:57.017 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max197.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max6639.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max6650.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/max6697.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/mcp3021.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/nct6683.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/nct6775-core.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/nct6775.ko 2023-07-14T10:39:57.018 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/nct7802.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pc87360.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pc87427.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pcf8591.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/adm1275.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/lm25066.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/ltc2978.ko 2023-07-14T10:39:57.019 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/max16064.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/max34440.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/max8688.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/pmbus.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/pmbus_core.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/tps40422.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/ucd9000.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/ucd9200.ko 2023-07-14T10:39:57.020 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/pmbus/zl6100.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/powr1220.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sch5627.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sch5636.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sch56xx-common.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sht15.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sht21.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/shtc1.ko 2023-07-14T10:39:57.021 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/sis5595.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/smsc47b397.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/smsc47m1.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/smsc47m192.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/thmc50.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/tmp102.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/tmp103.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/tmp401.ko 2023-07-14T10:39:57.022 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/tmp421.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/via-cputemp.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/via686a.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/vt1211.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/vt8231.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83627ehf.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83627hf.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83781d.ko 2023-07-14T10:39:57.023 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83791d.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83792d.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83793.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83795.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83l785ts.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/hwmon/w83l786ng.ko 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c 2023-07-14T10:39:57.024 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/algos 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/algos/i2c-algo-bit.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/algos/i2c-algo-pca.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-amd756-s4882.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-amd756.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-amd8111.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-ccgx-ucsi.ko 2023-07-14T10:39:57.025 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-designware-core.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-designware-pci.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-designware-platform.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-diolan-u2c.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-i801.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-isch.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-ismt.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-nforce2-s4985.ko 2023-07-14T10:39:57.026 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-nforce2.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-parport.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-pca-platform.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-piix4.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-scmi.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-simtec.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-sis96x.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-tiny-usb.ko 2023-07-14T10:39:57.027 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-via.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-viapro.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/busses/i2c-viperboard.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/i2c-dev.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/i2c-mux.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/i2c-slave-eeprom.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/i2c-smbus.ko 2023-07-14T10:39:57.028 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/i2c/i2c-stub.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/ib_cm.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/ib_core.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/ib_umad.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/ib_uverbs.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/iw_cm.ko 2023-07-14T10:39:57.029 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/rdma_cm.ko 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/core/rdma_ucm.ko 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/cxgb4 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/cxgb4/iw_cxgb4.ko 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mlx4 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mlx4/mlx4_ib.ko 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mlx5 2023-07-14T10:39:57.030 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mlx5/mlx5_ib.ko 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mthca 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/hw/mthca/ib_mthca.ko 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/ipoib 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/ipoib/ib_ipoib.ko 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/iser 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/iser/ib_iser.ko 2023-07-14T10:39:57.031 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/isert 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/isert/ib_isert.ko 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/srp 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/srp/ib_srp.ko 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/srpt 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/infiniband/ulp/srpt/ib_srpt.ko 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport 2023-07-14T10:39:57.032 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport/emu10k1-gp.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport/fm801-gp.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport/gameport.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport/lightning.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/gameport/ns558.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/joydev.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/keyboard 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/keyboard/gpio_keys.ko 2023-07-14T10:39:57.033 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/appletouch.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/bcm5974.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/cyapatp.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/elan_i2c.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/sermouse.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/synaptics_i2c.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/synaptics_usb.ko 2023-07-14T10:39:57.034 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/mouse/vsxxxaa.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/rmi4 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/rmi4/rmi_core.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/serio 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/serio/altera_ps2.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/serio/arc_ps2.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/serio/hyperv-keyboard.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/serio/serio_raw.ko 2023-07-14T10:39:57.035 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/input/sparse-keymap.ko 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/iommu 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/iommu/amd 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/iommu/amd/iommu_v2.ko 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/leds-blinkm.ko 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/leds-lm3530.ko 2023-07-14T10:39:57.036 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/leds-lp3944.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/leds-ss4200.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-audio.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-backlight.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-camera.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-default-on.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-heartbeat.ko 2023-07-14T10:39:57.037 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-oneshot.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-timer.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/leds/trigger/ledtrig-transient.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/bcache 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/bcache/bcache.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-bio-prison.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-cache-smq.ko 2023-07-14T10:39:57.038 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-cache.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-crypt.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-delay.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-flakey.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-log-userspace.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-multipath.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-queue-length.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-raid.ko 2023-07-14T10:39:57.039 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-round-robin.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-service-time.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-switch.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-thin-pool.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/dm-verity.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/faulty.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/linear.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/multipath.ko 2023-07-14T10:39:57.040 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/persistent-data 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/persistent-data/dm-persistent-data.ko 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/raid0.ko 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/raid1.ko 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/raid10.ko 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/md/raid456.ko 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion 2023-07-14T10:39:57.041 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptbase.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptctl.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptfc.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptlan.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptsas.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptscsih.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/message/fusion/mptspi.ko 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd 2023-07-14T10:39:57.042 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/lpc_ich.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/lpc_sch.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/mfd-core.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/sm501.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/viperboard.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/vx855.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/mfd/wl1273-core.ko 2023-07-14T10:39:57.043 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/altera-stapl 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/altera-stapl/altera-stapl.ko 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/apds9802als.ko 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/apds990x.ko 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/bh1770glc.ko 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/cb710 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/cb710/cb710.ko 2023-07-14T10:39:57.044 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/eeprom 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/eeprom/at24.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/eeprom/eeprom.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/eeprom/eeprom_93cx6.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/eeprom/max6875.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/enclosure.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/hpilo.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/isl29003.ko 2023-07-14T10:39:57.045 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/isl29020.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/lis3lv02d 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/lis3lv02d/lis3lv02d.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/lis3lv02d/lis3lv02d_i2c.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/mei 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/mei/mei-me.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/mei/mei-txe.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/mei/mei.ko 2023-07-14T10:39:57.046 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-gru 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-gru/gru.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-xp 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-xp/xp.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-xp/xpc.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/sgi-xp/xpnet.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/tifm_7xx1.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/tifm_core.ko 2023-07-14T10:39:57.047 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/tsl2550.ko 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/vmw_balloon.ko 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/vmw_vmci 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/misc/vmw_vmci/vmw_vmci.ko 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/bonding 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/bonding/bonding.ko 2023-07-14T10:39:57.048 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/b53 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/b53/b53_common.ko 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/bcm-sf2.ko 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/mv88e6060.ko 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/mv88e6xxx 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dsa/mv88e6xxx/mv88e6xxx.ko 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/dummy.ko 2023-07-14T10:39:57.049 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/eql.ko 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/3com 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/3com/3c574_cs.ko 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/3com/3c589_cs.ko 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/3com/3c59x.ko 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/3com/typhoon.ko 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/8390 2023-07-14T10:39:57.050 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/8390/8390.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/8390/axnet_cs.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/8390/ne2k-pci.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/8390/pcnet_cs.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/adaptec 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/adaptec/starfire.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/agere 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/agere/et131x.ko 2023-07-14T10:39:57.051 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/alteon 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/alteon/acenic.ko 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/altera 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/altera/altera_tse.ko 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/amd 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/amd/amd8111e.ko 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/amd/nmclan_cs.ko 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/amd/pcnet32.ko 2023-07-14T10:39:57.052 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/alx 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/alx/alx.ko 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atl1c 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atl1c/atl1c.ko 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atl1e 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atl1e/atl1e.ko 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atlx 2023-07-14T10:39:57.053 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atlx/atl1.ko 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/atheros/atlx/atl2.ko 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/b44.ko 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/bnx2.ko 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/bnx2x 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x.ko 2023-07-14T10:39:57.054 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/cnic.ko 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/genet 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/genet/genet.ko 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/broadcom/tg3.ko 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/brocade 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/brocade/bna 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/brocade/bna/bna.ko 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio 2023-07-14T10:39:57.055 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb/cxgb.ko 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb3 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb3/cxgb3.ko 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb4 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb4/cxgb4.ko 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb4vf 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf.ko 2023-07-14T10:39:57.056 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/libcxgb 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/chelsio/libcxgb/libcxgb.ko 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/cisco 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/cisco/enic 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/cisco/enic/enic.ko 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/de2104x.ko 2023-07-14T10:39:57.057 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/dmfe.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/tulip.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/uli526x.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/winbond-840.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dec/tulip/xircom_cb.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dlink 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dlink/dl2k.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dlink/sundance.ko 2023-07-14T10:39:57.058 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/dnet.ko 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/emulex 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/emulex/benet 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/emulex/benet/be2net.ko 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/ethoc.ko 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/fealnx.ko 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/e100.ko 2023-07-14T10:39:57.059 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/e1000 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/e1000/e1000.ko 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/e1000e 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/e1000e/e1000e.ko 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/fm10k 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/fm10k/fm10k.ko 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/i40e 2023-07-14T10:39:57.060 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/i40e/i40e.ko 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/iavf 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/iavf/iavf.ko 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/igb 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/igb/igb.ko 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/igbvf 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/igbvf/igbvf.ko 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/ixgbe 2023-07-14T10:39:57.061 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/ixgbe/ixgbe.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/ixgbevf 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/intel/ixgbevf/ixgbevf.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/jme.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/marvell 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/marvell/mvmdio.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/marvell/skge.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/marvell/sky2.ko 2023-07-14T10:39:57.062 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx4 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_core.ko 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx4/mlx4_en.ko 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx5 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx5/core 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.ko 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/micrel 2023-07-14T10:39:57.063 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/micrel/ksz884x.ko 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/myricom 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/myricom/myri10ge 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/myricom/myri10ge/myri10ge.ko 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/natsemi 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/natsemi/natsemi.ko 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/natsemi/ns83820.ko 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/neterion 2023-07-14T10:39:57.064 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/neterion/s2io.ko 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/nvidia 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/nvidia/forcedeth.ko 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/packetengines 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/packetengines/hamachi.ko 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/packetengines/yellowfin.ko 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic/netxen 2023-07-14T10:39:57.065 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic/netxen/netxen_nic.ko 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic/qla3xxx.ko 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic/qlcnic 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/qlogic/qlcnic/qlcnic.ko 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/rdc 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/rdc/r6040.ko 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/realtek 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/realtek/8139cp.ko 2023-07-14T10:39:57.066 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/realtek/8139too.ko 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/realtek/atp.ko 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/realtek/r8169.ko 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/rocker 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/rocker/rocker.ko 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sfc 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sfc/sfc.ko 2023-07-14T10:39:57.067 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/silan 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/silan/sc92031.ko 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sis 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sis/sis190.ko 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sis/sis900.ko 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/smsc 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/smsc/epic100.ko 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/smsc/smc91c92_cs.ko 2023-07-14T10:39:57.068 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/smsc/smsc911x.ko 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/smsc/smsc9420.ko 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/stmicro 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/stmicro/stmmac 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.ko 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/stmicro/stmmac/stmmac.ko 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sun 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sun/cassini.ko 2023-07-14T10:39:57.069 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sun/niu.ko 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sun/sungem.ko 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/sun/sunhme.ko 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/tehuti 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/tehuti/tehuti.ko 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/ti 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/ti/tlan.ko 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/via 2023-07-14T10:39:57.070 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/via/via-rhine.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/via/via-velocity.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/wiznet 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/wiznet/w5100.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/wiznet/w5300.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/xircom 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ethernet/xircom/xirc2ps_cs.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/geneve.ko 2023-07-14T10:39:57.071 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/hyperv 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/hyperv/hv_netvsc.ko 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ieee802154 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ieee802154/fakelb.ko 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ifb.ko 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ipvlan 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ipvlan/ipvlan.ko 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/macvlan.ko 2023-07-14T10:39:57.072 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/macvtap.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mdio 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mdio.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mdio/mdio-bcm-unimac.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mdio/mdio-bitbang.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mdio/mdio-regmap.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/mii.ko 2023-07-14T10:39:57.073 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/net_failover.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/netconsole.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/nlmon.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/pcs 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/pcs/pcs-lynx.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/pcs/pcs_xpcs.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/amd.ko 2023-07-14T10:39:57.074 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/bcm-phy-lib.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/bcm-phy-ptp.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/bcm7xxx.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/bcm87xx.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/broadcom.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/cicada.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/davicom.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/dp83640.ko 2023-07-14T10:39:57.075 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/et1011c.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/icplus.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/lxt.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/marvell.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/micrel.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/national.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/phylink.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/qsemi.ko 2023-07-14T10:39:57.076 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/realtek.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/smsc.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/ste10Xp.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/phy/vitesse.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/bsd_comp.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/ppp_async.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/ppp_deflate.ko 2023-07-14T10:39:57.077 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/ppp_generic.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/ppp_mppe.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/ppp_synctty.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/pppoe.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/pppox.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/ppp/pptp.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/slip 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/slip/slhc.ko 2023-07-14T10:39:57.078 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/slip/slip.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/sungem_phy.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/tap.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team_mode_activebackup.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team_mode_broadcast.ko 2023-07-14T10:39:57.079 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team_mode_loadbalance.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team_mode_random.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/team/team_mode_roundrobin.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/tun.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/veth.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/virtio_net.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/vmxnet3 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/vmxnet3/vmxnet3.ko 2023-07-14T10:39:57.080 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/vxlan 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/vxlan/vxlan.ko 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/xen-netback 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/xen-netback/xen-netback.ko 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/net/xen-netfront.ko 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ntb 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ntb/ntb.ko 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/nvme 2023-07-14T10:39:57.081 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/nvme/host 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/nvme/host/nvme-core.ko 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/nvme/host/nvme.ko 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/parport 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/parport/parport.ko 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/parport/parport_cs.ko 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/parport/parport_pc.ko 2023-07-14T10:39:57.082 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/parport/parport_serial.ko 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci/hotplug 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci/hotplug/acpiphp_ibm.ko 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci/pcie 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci/pcie/aer_inject.ko 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pci/xen-pcifront.ko 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pcmcia 2023-07-14T10:39:57.083 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pcmcia/i82092.ko 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pcmcia/pd6729.ko 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pcmcia/yenta_socket.ko 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pinctrl 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pinctrl/intel 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pinctrl/intel/pinctrl-cherryview.ko 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86 2023-07-14T10:39:57.084 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/acer-wmi.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/acerhdf.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/apple-gmux.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/asus-laptop.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/asus-nb-wmi.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/asus-wmi.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/classmate-laptop.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/eeepc-laptop.ko 2023-07-14T10:39:57.085 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/eeepc-wmi.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/fujitsu-laptop.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/fujitsu-tablet.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/hdaps.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/intel 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/intel/intel-rst.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/intel_ips.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/msi-wmi.ko 2023-07-14T10:39:57.086 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/mxm-wmi.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/panasonic-laptop.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/samsung-laptop.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/samsung-q10.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/topstar-laptop.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/toshiba_bluetooth.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/toshiba_haps.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/wmi-bmof.ko 2023-07-14T10:39:57.087 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/platform/x86/wmi.ko 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/powercap 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/powercap/intel_rapl_common.ko 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/powercap/intel_rapl_msr.ko 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps/clients 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps/clients/pps-gpio.ko 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps/clients/pps-ldisc.ko 2023-07-14T10:39:57.088 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps/clients/pps_parport.ko 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/pps/pps_core.ko 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ptp 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ptp/ptp.ko 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ptp/ptp_kvm.ko 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-bq32k.ko 2023-07-14T10:39:57.089 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-bq4802.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1286.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1307.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1374.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1511.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1553.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1672.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds1742.ko 2023-07-14T10:39:57.090 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds2404.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-ds3232.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-em3027.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-fm3130.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-isl12022.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-isl1208.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-m41t80.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-m48t35.ko 2023-07-14T10:39:57.091 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-m48t59.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-max6900.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-msm6242.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-pcf2127.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-pcf85063.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-pcf8523.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-pcf8563.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-pcf8583.ko 2023-07-14T10:39:57.092 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-rp5c01.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-rs5c372.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-rv3029c2.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-rx8025.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-rx8581.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-stk17ta8.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/rtc/rtc-x1205.ko 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi 2023-07-14T10:39:57.093 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/3w-9xxx.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/3w-sas.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/3w-xxxx.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/BusLogic.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/a100u2w.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/aacraid 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/aacraid/aacraid.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/advansys.ko 2023-07-14T10:39:57.094 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/aic7xxx 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/aic7xxx/aic79xx.ko 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/aic7xxx/aic7xxx.ko 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/am53c974.ko 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/arcmsr 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/arcmsr/arcmsr.ko 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/atp870u.ko 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/be2iscsi 2023-07-14T10:39:57.095 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/be2iscsi/be2iscsi.ko 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bfa 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bfa/bfa.ko 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bnx2fc 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bnx2fc/bnx2fc.ko 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bnx2i 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/bnx2i/bnx2i.ko 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/ch.ko 2023-07-14T10:39:57.096 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/csiostor 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/csiostor/csiostor.ko 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi/cxgb3i 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi/cxgb3i/cxgb3i.ko 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi/cxgb4i 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi/cxgb4i/cxgb4i.ko 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/cxgbi/libcxgbi.ko 2023-07-14T10:39:57.097 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/dc395x.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/device_handler 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/device_handler/scsi_dh_alua.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/device_handler/scsi_dh_emc.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/device_handler/scsi_dh_hp_sw.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/device_handler/scsi_dh_rdac.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/dmx3191d.ko 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/esas2r 2023-07-14T10:39:57.098 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/esas2r/esas2r.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/esp_scsi.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/fcoe 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/fcoe/fcoe.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/fcoe/libfcoe.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/fnic 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/fnic/fnic.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/hpsa.ko 2023-07-14T10:39:57.099 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/hptiop.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/hv_storvsc.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/initio.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/ipr.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/ips.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/isci 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/isci/isci.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/iscsi_boot_sysfs.ko 2023-07-14T10:39:57.100 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/iscsi_tcp.ko 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libfc 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libfc/libfc.ko 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libiscsi.ko 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libiscsi_tcp.ko 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libsas 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/libsas/libsas.ko 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/lpfc 2023-07-14T10:39:57.101 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/lpfc/lpfc.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/megaraid 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/megaraid.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/megaraid/megaraid_mbox.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/megaraid/megaraid_mm.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/megaraid/megaraid_sas.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/mpt3sas 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/mpt3sas/mpt3sas.ko 2023-07-14T10:39:57.102 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/mvsas 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/mvsas/mvsas.ko 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/mvumi.ko 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/pm8001 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/pm8001/pm80xx.ko 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/pmcraid.ko 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla1280.ko 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla2xxx 2023-07-14T10:39:57.103 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla2xxx/qla2xxx.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla2xxx/tcm_qla2xxx.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla4xxx 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/qla4xxx/qla4xxx.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/raid_class.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_debug.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_transport_fc.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_transport_iscsi.ko 2023-07-14T10:39:57.104 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_transport_sas.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_transport_spi.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/scsi_transport_srp.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/ses.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/st.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/stex.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/sym53c8xx_2 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/sym53c8xx_2/sym53c8xx.ko 2023-07-14T10:39:57.105 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/virtio_scsi.ko 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/vmw_pvscsi.ko 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/scsi/wd719x.ko 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ssb 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ssb/ssb.ko 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/iscsi 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/iscsi/iscsi_target_mod.ko 2023-07-14T10:39:57.106 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/loopback 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/loopback/tcm_loop.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/target_core_file.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/target_core_iblock.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/target_core_mod.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/target_core_pscsi.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/target_core_user.ko 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/tcm_fc 2023-07-14T10:39:57.107 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/target/tcm_fc/tcm_fc.ko 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/acpi_thermal_rel.ko 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/int3400_thermal.ko 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/int3401_thermal.ko 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/int3402_thermal.ko 2023-07-14T10:39:57.108 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/int3403_thermal.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/int340x_thermal_zone.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_mbox.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.ko 2023-07-14T10:39:57.109 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/intel_soc_dts_iosf.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/intel_soc_dts_thermal.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/thermal/intel/x86_pkg_temp_thermal.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/ipwireless 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/ipwireless/ipwireless.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/n_gsm.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/n_hdlc.ko 2023-07-14T10:39:57.110 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/nozomi.ko 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial/8250 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial/8250/serial_cs.ko 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial/arc_uart.ko 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial/jsm 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/serial/jsm/jsm.ko 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/tty/synclink_gt.ko 2023-07-14T10:39:57.111 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ufs 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ufs/core 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ufs/core/ufshcd-core.ko 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ufs/host 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/ufs/host/ufshcd-pci.ko 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio/uio.ko 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio/uio_aec.ko 2023-07-14T10:39:57.112 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio/uio_cif.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio/uio_pci_generic.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/uio/uio_sercos3.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/class 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/class/cdc-acm.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/class/cdc-wdm.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/class/usblp.ko 2023-07-14T10:39:57.113 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/class/usbtmc.ko 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/host 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/host/sl811-hcd.ko 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/image 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/image/mdc800.ko 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/image/microtek.ko 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/adutux.ko 2023-07-14T10:39:57.114 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/appledisplay.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/emi26.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/emi62.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/ezusb.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/idmouse.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/iowarrior.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/isight_firmware.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/ldusb.ko 2023-07-14T10:39:57.115 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/legousbtower.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/sisusbvga 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/sisusbvga/sisusbvga.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/trancevibrator.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/usb3503.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/usblcd.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/usbsevseg.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/uss720.ko 2023-07-14T10:39:57.116 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/misc/yurex.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/phy 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/phy/phy-generic.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/aircable.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ark3116.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/belkin_sa.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ch341.ko 2023-07-14T10:39:57.117 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/cp210x.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/cyberjack.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/cypress_m8.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/digi_acceleport.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/empeg.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ftdi_sio.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/garmin_gps.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/io_edgeport.ko 2023-07-14T10:39:57.118 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/io_ti.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ipaq.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ipw.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ir-usb.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/iuu_phoenix.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/keyspan.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/keyspan_pda.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/kl5kusb105.ko 2023-07-14T10:39:57.119 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/kobil_sct.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/mct_u232.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/mos7720.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/mos7840.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/navman.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/omninet.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/opticon.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/option.ko 2023-07-14T10:39:57.120 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/oti6858.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/pl2303.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/qcaux.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/qcserial.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/quatech2.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/safe_serial.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/sierra.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/spcp8x5.ko 2023-07-14T10:39:57.121 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ssu100.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/symbolserial.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/ti_usb_3410_5052.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/usb-serial-simple.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/usb_debug.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/usb_wwan.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/visor.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/whiteheat.ko 2023-07-14T10:39:57.122 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/serial/xsens_mt.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/uas.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-alauda.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-cypress.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-datafab.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-eneub6250.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-freecom.ko 2023-07-14T10:39:57.123 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-isd200.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-jumpshot.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-karma.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-onetouch.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-realtek.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-sddr09.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-sddr55.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/ums-usbat.ko 2023-07-14T10:39:57.124 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/storage/usb-storage.ko 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/usbip 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/usbip/usbip-core.ko 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/usbip/usbip-host.ko 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/usb/usbip/vhci-hcd.ko 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio/pci 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio/pci/vfio-pci-core.ko 2023-07-14T10:39:57.125 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio/pci/vfio-pci.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio/vfio.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vfio/vfio_iommu_type1.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vhost 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vhost/vhost.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vhost/vhost_iotlb.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vhost/vhost_net.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/vhost/vhost_scsi.ko 2023-07-14T10:39:57.126 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/video 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/video/backlight 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/video/backlight/apple_bl.ko 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/video/backlight/lcd.ko 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/video/backlight/platform_lcd.ko 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio.ko 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_balloon.ko 2023-07-14T10:39:57.127 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_mmio.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_pci.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_pci_legacy_dev.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_pci_modern_dev.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/virtio/virtio_ring.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/masters 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/masters/ds2482.ko 2023-07-14T10:39:57.128 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/masters/ds2490.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2406.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2408.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2413.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2423.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2431.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2433.ko 2023-07-14T10:39:57.129 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2780.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds2781.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_ds28e04.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_smem.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/slaves/w1_therm.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/w1/wire.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/alim1535_wdt.ko 2023-07-14T10:39:57.130 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/alim7101_wdt.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/f71808e_wdt.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/hpwdt.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/i6300esb.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/iTCO_vendor_support.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/iTCO_wdt.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/ib700wdt.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/ibmasr.ko 2023-07-14T10:39:57.131 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/ie6xx_wdt.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/it8712f_wdt.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/it87_wdt.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/machzwd.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/nv_tco.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/pcwd_pci.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/pcwd_usb.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/sbc_fitpc2_wdt.ko 2023-07-14T10:39:57.132 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/sch311x_wdt.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/softdog.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/sp5100_tco.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/via_wdt.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/w83627hf_wdt.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/w83877f_wdt.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/w83977f_wdt.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/wdt_pci.ko 2023-07-14T10:39:57.133 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/watchdog/xen_wdt.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-acpi-processor.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-evtchn.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-gntalloc.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-gntdev.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-pciback 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-pciback/xen-pciback.ko 2023-07-14T10:39:57.134 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xen-privcmd.ko 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xenfs 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/drivers/xen/xenfs/xenfs.ko 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/9p 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/9p/9p.ko 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/affs 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/affs/affs.ko 2023-07-14T10:39:57.135 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/befs 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/befs/befs.ko 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/binfmt_misc.ko 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/btrfs 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/btrfs/btrfs.ko 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/cachefiles 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/cachefiles/cachefiles.ko 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ceph 2023-07-14T10:39:57.136 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ceph/ceph.ko 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/coda 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/coda/coda.ko 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/cramfs 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/cramfs/cramfs.ko 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/dlm 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/dlm/dlm.ko 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ecryptfs 2023-07-14T10:39:57.137 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ecryptfs/ecryptfs.ko 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/f2fs 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/f2fs/f2fs.ko 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fat 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fat/fat.ko 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fat/msdos.ko 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fat/vfat.ko 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fscache 2023-07-14T10:39:57.138 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fscache/fscache.ko 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fuse 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fuse/cuse.ko 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/fuse/fuse.ko 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/gfs2 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/gfs2/gfs2.ko 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/hfs 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/hfs/hfs.ko 2023-07-14T10:39:57.139 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/hfsplus 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/hfsplus/hfsplus.ko 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/isofs 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/isofs/isofs.ko 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/jfs 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/jfs/jfs.ko 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/lockd 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/lockd/lockd.ko 2023-07-14T10:39:57.140 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/minix 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/minix/minix.ko 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/netfs 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/netfs/netfs.ko 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/blocklayout 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/blocklayout/blocklayoutdriver.ko 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/filelayout 2023-07-14T10:39:57.141 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/filelayout/nfs_layout_nfsv41_files.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/flexfilelayout 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/flexfilelayout/nfs_layout_flexfiles.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/nfs.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/nfsv3.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs/nfsv4.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs_common 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs_common/grace.ko 2023-07-14T10:39:57.142 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfs_common/nfs_acl.ko 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfsd 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nfsd/nfsd.ko 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nilfs2 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nilfs2/nilfs2.ko 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-celtic.ko 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-centeuro.ko 2023-07-14T10:39:57.143 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-croatian.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-cyrillic.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-gaelic.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-greek.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-iceland.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-inuit.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-roman.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-romanian.ko 2023-07-14T10:39:57.144 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/mac-turkish.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp1250.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp1251.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp1255.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp737.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp775.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp850.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp852.ko 2023-07-14T10:39:57.145 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp855.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp857.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp860.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp861.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp862.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp863.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp864.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp865.ko 2023-07-14T10:39:57.146 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp866.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp869.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp874.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp932.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp936.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp949.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_cp950.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_euc-jp.ko 2023-07-14T10:39:57.147 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-1.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-13.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-14.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-15.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-2.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-3.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-4.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-5.ko 2023-07-14T10:39:57.148 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-6.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-7.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_iso8859-9.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_koi8-r.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_koi8-ru.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_koi8-u.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/nls/nls_utf8.ko 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2 2023-07-14T10:39:57.149 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/cluster 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/cluster/ocfs2_nodemanager.ko 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/dlm 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/dlm/ocfs2_dlm.ko 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/dlmfs 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/dlmfs/ocfs2_dlmfs.ko 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/ocfs2.ko 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/ocfs2_stack_o2cb.ko 2023-07-14T10:39:57.150 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/ocfs2_stack_user.ko 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ocfs2/ocfs2_stackglue.ko 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/overlayfs 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/overlayfs/overlay.ko 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/pstore 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/pstore/ramoops.ko 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/reiserfs 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/reiserfs/reiserfs.ko 2023-07-14T10:39:57.151 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/romfs 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/romfs/romfs.ko 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb/client 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb/client/cifs.ko 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb/common 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb/common/cifs_arc4.ko 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/smb/common/cifs_md4.ko 2023-07-14T10:39:57.152 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/squashfs 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/squashfs/squashfs.ko 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/sysv 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/sysv/sysv.ko 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/udf 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/udf/udf.ko 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ufs 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/ufs/ufs.ko 2023-07-14T10:39:57.153 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/xfs 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/fs/xfs/xfs.ko 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel/rcu 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel/rcu/rcutorture.ko 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel/torture.ko 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel/trace 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/kernel/trace/ring_buffer_benchmark.ko 2023-07-14T10:39:57.154 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/asn1_encoder.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/crc-itu-t.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/crc8.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/crypto 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/crypto/libarc4.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/crypto/libdes.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/libcrc32c.ko 2023-07-14T10:39:57.155 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/lru_cache.ko 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/lz4 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/lz4/lz4_compress.ko 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/lz4/lz4hc_compress.ko 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/math 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/math/cordic.ko 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/raid6 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/raid6/raid6_pq.ko 2023-07-14T10:39:57.156 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/reed_solomon 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/reed_solomon/reed_solomon.ko 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/ts_bm.ko 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/ts_fsm.ko 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/ts_kmp.ko 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/zstd 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/lib/zstd/zstd_compress.ko 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/mm 2023-07-14T10:39:57.157 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/mm/hwpoison-inject.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/6lowpan.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_dest.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_fragment.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_hop.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_ipv6.ko 2023-07-14T10:39:57.158 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_mobility.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_routing.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/6lowpan/nhc_udp.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802/garp.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802/mrp.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802/p8022.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802/psnap.ko 2023-07-14T10:39:57.159 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/802/stp.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/8021q 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/8021q/8021q.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/9p 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/9p/9pnet.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/9p/9pnet_fd.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/9p/9pnet_rdma.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/9p/9pnet_virtio.ko 2023-07-14T10:39:57.160 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/batman-adv 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/batman-adv/batman-adv.ko 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/br_netfilter.ko 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/bridge.ko 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_802_3.ko 2023-07-14T10:39:57.161 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_among.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_arp.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_arpreply.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_dnat.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_ip.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_ip6.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_limit.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_log.ko 2023-07-14T10:39:57.162 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_mark.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_mark_m.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_nflog.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_pkttype.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_redirect.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_snat.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_stp.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebt_vlan.ko 2023-07-14T10:39:57.163 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebtable_broute.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebtable_filter.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebtable_nat.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/ebtables.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/nf_conntrack_bridge.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/nft_meta_bridge.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/bridge/netfilter/nft_reject_bridge.ko 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ceph 2023-07-14T10:39:57.164 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ceph/libceph.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/core 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/core/failover.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/core/pktgen.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dccp 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dccp/dccp.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dccp/dccp_diag.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dccp/dccp_ipv4.ko 2023-07-14T10:39:57.165 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dccp/dccp_ipv6.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dns_resolver 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dns_resolver/dns_resolver.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa/dsa_core.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa/tag_brcm.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa/tag_dsa.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa/tag_none.ko 2023-07-14T10:39:57.166 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/dsa/tag_trailer.ko 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ieee802154 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ieee802154/6lowpan 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ieee802154/6lowpan/ieee802154_6lowpan.ko 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ieee802154/ieee802154.ko 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ieee802154/ieee802154_socket.ko 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ah4.ko 2023-07-14T10:39:57.167 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/esp4.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/fou.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/gre.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/inet_diag.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ip_gre.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ip_tunnel.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ip_vti.ko 2023-07-14T10:39:57.168 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ipcomp.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/ipip.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/arp_tables.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/arpt_mangle.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/arptable_filter.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/ipt_ECN.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/ipt_SYNPROXY.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/ipt_ah.ko 2023-07-14T10:39:57.169 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/ipt_rpfilter.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/iptable_mangle.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/iptable_nat.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/iptable_raw.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/iptable_security.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_defrag_ipv4.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_dup_ipv4.ko 2023-07-14T10:39:57.170 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_nat_h323.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_nat_pptp.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_nat_snmp_basic.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_socket_ipv4.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nf_tproxy_ipv4.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nft_dup_ipv4.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nft_fib_ipv4.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/netfilter/nft_reject_ipv4.ko 2023-07-14T10:39:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_bic.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_dctcp.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_diag.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_highspeed.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_htcp.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_hybla.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_illinois.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_lp.ko 2023-07-14T10:39:57.172 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_scalable.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_vegas.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_veno.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_westwood.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tcp_yeah.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/tunnel4.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/udp_diag.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/udp_tunnel.ko 2023-07-14T10:39:57.173 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv4/xfrm4_tunnel.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/ah6.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/esp6.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/fou6.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/ip6_tunnel.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/ip6_udp_tunnel.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/ip6_vti.ko 2023-07-14T10:39:57.174 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/ipcomp6.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6_tables.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_REJECT.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_SYNPROXY.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_ah.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_eui64.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_frag.ko 2023-07-14T10:39:57.175 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_hbh.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_ipv6header.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_mh.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_rpfilter.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6t_rt.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6table_filter.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6table_mangle.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6table_nat.ko 2023-07-14T10:39:57.176 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6table_raw.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/ip6table_security.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nf_defrag_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nf_dup_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nf_reject_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nf_socket_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nf_tproxy_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nft_dup_ipv6.ko 2023-07-14T10:39:57.177 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nft_fib_ipv6.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/netfilter/nft_reject_ipv6.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/sit.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/tunnel6.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/ipv6/xfrm6_tunnel.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/key 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/key/af_key.ko 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp 2023-07-14T10:39:57.178 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_core.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_debugfs.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_eth.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_ip.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_ip6.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_netlink.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/l2tp/l2tp_ppp.ko 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/llc 2023-07-14T10:39:57.179 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/llc/llc.ko 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/mac802154 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/mac802154/mac802154.ko 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/mpls 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/mpls/mpls_gso.ko 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set.ko 2023-07-14T10:39:57.180 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_bitmap_ip.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_bitmap_ipmac.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_bitmap_port.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_ip.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_ipmark.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_ipport.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_ipportip.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_ipportnet.ko 2023-07-14T10:39:57.181 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_mac.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_net.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_netiface.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_netnet.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_netport.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_hash_netportnet.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipset/ip_set_list_set.ko 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs 2023-07-14T10:39:57.182 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_dh.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_fo.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_ftp.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_lblc.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_lblcr.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_lc.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_nq.ko 2023-07-14T10:39:57.183 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_pe_sip.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_rr.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_sed.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_sh.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_wlc.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/ipvs/ip_vs_wrr.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conncount.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack.ko 2023-07-14T10:39:57.184 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_amanda.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_broadcast.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_ftp.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_h323.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_irc.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_netbios_ns.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_netlink.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_pptp.ko 2023-07-14T10:39:57.185 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_sane.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_sip.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_snmp.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_conntrack_tftp.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_dup_netdev.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_flow_table.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_flow_table_inet.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_log_syslog.ko 2023-07-14T10:39:57.186 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat_amanda.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat_ftp.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat_irc.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat_sip.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_nat_tftp.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_synproxy_core.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nf_tables.ko 2023-07-14T10:39:57.187 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nfnetlink.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nfnetlink_acct.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nfnetlink_log.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nfnetlink_osf.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nfnetlink_queue.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_chain_nat.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_compat.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_ct.ko 2023-07-14T10:39:57.188 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_dup_netdev.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_fib.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_fwd_netdev.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_hash.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_limit.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_log.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_masq.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_nat.ko 2023-07-14T10:39:57.189 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_numgen.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_queue.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_quota.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_redir.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_reject.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_reject_inet.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_socket.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/nft_tunnel.ko 2023-07-14T10:39:57.190 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_AUDIT.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_CHECKSUM.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_CLASSIFY.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_CONNSECMARK.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_CT.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_DSCP.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_HL.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_HMARK.ko 2023-07-14T10:39:57.191 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_IDLETIMER.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_LED.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_LOG.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_MASQUERADE.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_NETMAP.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_NFLOG.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_NFQUEUE.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_RATEEST.ko 2023-07-14T10:39:57.192 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_REDIRECT.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_SECMARK.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_TCPMSS.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_TCPOPTSTRIP.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_TEE.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_TPROXY.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_TRACE.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_addrtype.ko 2023-07-14T10:39:57.193 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_bpf.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_cgroup.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_cluster.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_comment.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_connbytes.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_connlabel.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_connlimit.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_connmark.ko 2023-07-14T10:39:57.194 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_conntrack.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_cpu.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_dccp.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_devgroup.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_dscp.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_ecn.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_esp.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_hashlimit.ko 2023-07-14T10:39:57.195 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_helper.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_hl.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_ipcomp.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_iprange.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_ipvs.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_l2tp.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_length.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_limit.ko 2023-07-14T10:39:57.196 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_mac.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_mark.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_multiport.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_nat.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_nfacct.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_osf.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_owner.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_physdev.ko 2023-07-14T10:39:57.197 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_pkttype.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_policy.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_quota.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_rateest.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_realm.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_recent.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_sctp.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_set.ko 2023-07-14T10:39:57.198 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_socket.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_state.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_statistic.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_string.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_tcpmss.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_time.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netfilter/xt_u32.ko 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netlink 2023-07-14T10:39:57.199 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/netlink/netlink_diag.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/nsh 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/nsh/nsh.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/openvswitch 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/openvswitch/openvswitch.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/openvswitch/vport-geneve.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/openvswitch/vport-gre.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/openvswitch/vport-vxlan.ko 2023-07-14T10:39:57.200 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/packet 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/packet/af_packet_diag.ko 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/rds 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/rds/rds.ko 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/rds/rds_rdma.ko 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/rds/rds_tcp.ko 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_csum.ko 2023-07-14T10:39:57.201 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_gact.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_ipt.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_mirred.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_nat.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_pedit.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_police.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_simple.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_skbedit.ko 2023-07-14T10:39:57.202 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/act_vlan.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_basic.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_bpf.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_flow.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_fw.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_route.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/cls_u32.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_cmp.ko 2023-07-14T10:39:57.203 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_ipset.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_meta.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_nbyte.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_text.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/em_u32.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_choke.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_codel.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_drr.ko 2023-07-14T10:39:57.204 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_fq.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_gred.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_hfsc.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_hhf.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_htb.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_ingress.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_mqprio.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_mqprio_lib.ko 2023-07-14T10:39:57.205 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_multiq.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_netem.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_pie.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_plug.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_prio.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_qfq.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_red.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_sfb.ko 2023-07-14T10:39:57.206 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_sfq.ko 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_tbf.ko 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sched/sch_teql.ko 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sctp 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sctp/sctp.ko 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sctp/sctp_diag.ko 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/auth_gss 2023-07-14T10:39:57.207 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/auth_gss/auth_rpcgss.ko 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/auth_gss/rpcsec_gss_krb5.ko 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/sunrpc.ko 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/xprtrdma 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/sunrpc/xprtrdma/rpcrdma.ko 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/tipc 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/tipc/diag.ko 2023-07-14T10:39:57.208 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/tipc/tipc.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/unix 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/unix/unix_diag.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock/vmw_vsock_virtio_transport_common.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock/vmw_vsock_vmci_transport.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock/vsock.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock/vsock_diag.ko 2023-07-14T10:39:57.209 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/vmw_vsock/vsock_loopback.ko 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/xfrm 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/net/xfrm/xfrm_ipcomp.ko 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security/keys 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security/keys/encrypted-keys 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security/keys/encrypted-keys/encrypted-keys.ko 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security/keys/trusted-keys 2023-07-14T10:39:57.210 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/security/keys/trusted-keys/trusted.ko 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/virt 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/virt/lib 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/kernel/virt/lib/irqbypass.ko 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.alias 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.alias.bin 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.builtin 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.builtin.bin 2023-07-14T10:39:57.211 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.builtin.modinfo 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.dep 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.dep.bin 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.devname 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.order 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.softdep 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.symbols 2023-07-14T10:39:57.212 INFO:teuthology.orchestra.run.smithi040.stdout:/lib/modules/6.5.0-rc1-ga440c79eee02/modules.symbols.bin 2023-07-14T10:39:57.213 DEBUG:teuthology.task.kernel:get_image_version: 6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:57.213 DEBUG:teuthology.orchestra.run.smithi040:> rpm --scripts -qp /tmp/kernel.x86_64.rpm 2023-07-14T10:39:57.241 INFO:teuthology.orchestra.run.smithi040.stdout:postinstall scriptlet (using /bin/sh): 2023-07-14T10:39:57.241 INFO:teuthology.orchestra.run.smithi040.stdout:if [ -x /sbin/installkernel -a -r /boot/vmlinuz-6.5.0-rc1-ga440c79eee02 -a -r /boot/System.map-6.5.0-rc1-ga440c79eee02 ]; then 2023-07-14T10:39:57.241 INFO:teuthology.orchestra.run.smithi040.stdout:cp /boot/vmlinuz-6.5.0-rc1-ga440c79eee02 /boot/.vmlinuz-6.5.0-rc1-ga440c79eee02-rpm 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:cp /boot/System.map-6.5.0-rc1-ga440c79eee02 /boot/.System.map-6.5.0-rc1-ga440c79eee02-rpm 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:rm -f /boot/vmlinuz-6.5.0-rc1-ga440c79eee02 /boot/System.map-6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/installkernel 6.5.0-rc1-ga440c79eee02 /boot/.vmlinuz-6.5.0-rc1-ga440c79eee02-rpm /boot/.System.map-6.5.0-rc1-ga440c79eee02-rpm 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:rm -f /boot/.vmlinuz-6.5.0-rc1-ga440c79eee02-rpm /boot/.System.map-6.5.0-rc1-ga440c79eee02-rpm 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:fi 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:preuninstall scriptlet (using /bin/sh): 2023-07-14T10:39:57.242 INFO:teuthology.orchestra.run.smithi040.stdout:if [ -x /sbin/new-kernel-pkg ]; then 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:new-kernel-pkg --remove 6.5.0-rc1-ga440c79eee02 --rminitrd --initrdfile=/boot/initramfs-6.5.0-rc1-ga440c79eee02.img 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:elif [ -x /usr/bin/kernel-install ]; then 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-install remove 6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:fi 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:postuninstall scriptlet (using /bin/sh): 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:if [ -x /sbin/update-bootloader ]; then 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/update-bootloader --remove 6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:57.243 INFO:teuthology.orchestra.run.smithi040.stdout:fi 2023-07-14T10:39:57.244 DEBUG:teuthology.orchestra.run.smithi040:> sudo rpm -qi grub2-tools 2023-07-14T10:39:57.324 INFO:teuthology.orchestra.run.smithi040.stdout:Name : grub2-tools 2023-07-14T10:39:57.324 INFO:teuthology.orchestra.run.smithi040.stdout:Epoch : 1 2023-07-14T10:39:57.324 INFO:teuthology.orchestra.run.smithi040.stdout:Version : 2.02 2023-07-14T10:39:57.324 INFO:teuthology.orchestra.run.smithi040.stdout:Release : 123.el8 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:Architecture: x86_64 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:Install Date: Thu 14 Jul 2022 06:22:19 PM UTC 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:Group : System Environment/Base 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:Size : 9265450 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:License : GPLv3+ 2023-07-14T10:39:57.325 INFO:teuthology.orchestra.run.smithi040.stdout:Signature : RSA/SHA256, Thu 31 Mar 2022 07:24:35 PM UTC, Key ID 199e2f91fd431d51 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Source RPM : grub2-2.02-123.el8.src.rpm 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Build Date : Mon 28 Mar 2022 02:12:09 PM UTC 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Build Host : x86-vm-09.build.eng.bos.redhat.com 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Relocations : (not relocatable) 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Packager : Red Hat, Inc. 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Vendor : Red Hat, Inc. 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:URL : http://www.gnu.org/software/grub/ 2023-07-14T10:39:57.326 INFO:teuthology.orchestra.run.smithi040.stdout:Summary : Support tools for GRUB. 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout:Description : 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout:customizable bootloader with modular architecture. It supports a rich 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout:variety of kernel formats, file systems, computer architectures and 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout:hardware devices. 2023-07-14T10:39:57.327 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:39:57.328 INFO:teuthology.orchestra.run.smithi040.stdout:This subpackage provides tools for support of all platforms. 2023-07-14T10:39:57.329 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-07-14T10:39:57.329 INFO:teuthology.task.kernel:Updating grub on smithi040 to boot 6.5.0-rc1-ga440c79eee02 2023-07-14T10:39:57.329 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-07-14T10:39:59.110 INFO:teuthology.orchestra.run.smithi040.stderr:Generating grub configuration file ... 2023-07-14T10:40:01.272 INFO:teuthology.orchestra.run.smithi040.stderr:done 2023-07-14T10:40:01.274 DEBUG:teuthology.orchestra.run.smithi040:> mktemp 2023-07-14T10:40:01.296 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/tmp.Tq3rgVQauF 2023-07-14T10:40:01.296 DEBUG:teuthology.orchestra.run.smithi040:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Tq3rgVQauF 2023-07-14T10:40:01.368 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 0666 /tmp/tmp.Tq3rgVQauF 2023-07-14T10:40:01.511 DEBUG:teuthology.orchestra.remote:smithi040:/tmp/tmp.Tq3rgVQauF is 6KB 2023-07-14T10:40:01.558 DEBUG:teuthology.orchestra.run.smithi040:> rm -fr /tmp/tmp.Tq3rgVQauF 2023-07-14T10:40:01.575 DEBUG:teuthology.orchestra.run.smithi040:> sudo /bin/ls /boot/loader/entries || true 2023-07-14T10:40:01.646 INFO:teuthology.orchestra.run.smithi040.stdout:5a84457b7b52416e95d47d02e2e4a60f-0-rescue.conf 2023-07-14T10:40:01.646 INFO:teuthology.orchestra.run.smithi040.stdout:5a84457b7b52416e95d47d02e2e4a60f-4.18.0-372.9.1.el8.x86_64.conf 2023-07-14T10:40:01.646 INFO:teuthology.orchestra.run.smithi040.stdout:5a84457b7b52416e95d47d02e2e4a60f-6.5.0-rc1-ga440c79eee02.conf 2023-07-14T10:40:01.647 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-set-default 5a84457b7b52416e95d47d02e2e4a60f-6.5.0-rc1-ga440c79eee02 2023-07-14T10:40:01.807 DEBUG:teuthology.orchestra.run.smithi040:> sudo shutdown -r now 2023-07-14T10:40:31.810 INFO:teuthology.task.kernel:Checking client client.0 for new kernel version... 2023-07-14T10:40:31.811 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:40:31.812 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:40:50.288 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.40 2023-07-14T10:40:59.290 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:40:59.291 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:41:59.294 DEBUG:teuthology.orchestra.remote:timed out 2023-07-14T10:42:11.295 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:42:11.296 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:42:11.662 DEBUG:teuthology.orchestra.run.smithi040:> true 2023-07-14T10:42:12.064 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:42:12.065 INFO:teuthology.task.kernel:Checking kernel version of client.0, want "6.5.0-rc1-ga440c79eee02"... 2023-07-14T10:42:12.065 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2023-07-14T10:42:12.108 INFO:teuthology.orchestra.run.smithi040.stdout:6.5.0-rc1-ga440c79eee02 2023-07-14T10:42:12.109 DEBUG:teuthology.task.kernel:current kernel version is 6.5.0-rc1-ga440c79eee02 vs 6.5.0-rc1-ga440c79eee02 2023-07-14T10:42:12.109 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-07-14T10:42:12.109 DEBUG:teuthology.task.kernel:Distro of this test job: rhel 2023-07-14T10:42:13.110 DEBUG:teuthology.parallel:result is None 2023-07-14T10:42:13.111 INFO:teuthology.run_tasks:Running task internal.base... 2023-07-14T10:42:13.120 INFO:teuthology.task.internal:Creating test directory... 2023-07-14T10:42:13.121 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-07-14T10:42:13.126 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-07-14T10:42:13.129 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-07-14T10:42:13.169 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-07-14T10:42:13.176 INFO:teuthology.run_tasks:Running task internal.archive... 2023-07-14T10:42:13.190 INFO:teuthology.task.internal:Creating archive directory... 2023-07-14T10:42:13.190 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-07-14T10:42:13.214 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-07-14T10:42:13.217 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-07-14T10:42:13.274 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-07-14T10:42:13.281 INFO:teuthology.task.internal:Enabling coredump saving... 2023-07-14T10:42:13.282 DEBUG:teuthology.orchestra.run.smithi045:> test -f /run/.containerenv -o -f /.dockerenv 2023-07-14T10:42:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:42:13.306 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2023-07-14T10:42:13.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:42:13.326 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-07-14T10:42:13.330 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-07-14T10:42:13.349 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-07-14T10:42:13.393 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.412 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.415 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.435 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.444 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.481 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-07-14T10:42:13.484 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-07-14T10:42:13.492 INFO:teuthology.task.internal:Configuring sudo... 2023-07-14T10:42:13.492 DEBUG:teuthology.orchestra.run.smithi040:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-07-14T10:42:13.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-07-14T10:42:13.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-07-14T10:42:13.562 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-07-14T10:42:13.572 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-07-14T10:42:13.573 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-07-14T10:42:13.577 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-07-14T10:42:13.580 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-07-14T10:42:13.615 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:13.732 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:13.802 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:13.876 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:13.956 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T10:42:13.957 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-07-14T10:42:14.025 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:14.076 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:14.130 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:14.222 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:14.265 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T10:42:14.265 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-07-14T10:42:14.336 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:14.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-07-14T10:42:14.436 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:14.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-07-14T10:42:14.567 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T10:42:14.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-07-14T10:42:14.636 DEBUG:teuthology.orchestra.run.smithi040:> sudo service rsyslog restart 2023-07-14T10:42:14.640 DEBUG:teuthology.orchestra.run.smithi045:> sudo service rsyslog restart 2023-07-14T10:42:14.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2023-07-14T10:42:14.706 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T10:42:14.741 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T10:42:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T10:42:15.073 INFO:teuthology.run_tasks:Running task internal.timer... 2023-07-14T10:42:15.081 INFO:teuthology.task.internal:Starting timer... 2023-07-14T10:42:15.081 INFO:teuthology.run_tasks:Running task pcp... 2023-07-14T10:42:15.098 INFO:teuthology.run_tasks:Running task selinux... 2023-07-14T10:42:15.108 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-07-14T10:42:15.109 DEBUG:teuthology.orchestra.run.smithi040:> sudo service auditd rotate 2023-07-14T10:42:15.226 INFO:teuthology.orchestra.run.smithi040.stdout:Rotating logs: 2023-07-14T10:42:15.229 DEBUG:teuthology.orchestra.run.smithi045:> sudo service auditd rotate 2023-07-14T10:42:15.290 INFO:teuthology.orchestra.run.smithi045.stdout:Rotating logs: 2023-07-14T10:42:15.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo service auditd rotate 2023-07-14T10:42:15.357 INFO:teuthology.orchestra.run.smithi130.stdout:Rotating logs: 2023-07-14T10:42:15.359 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-07-14T10:42:15.359 DEBUG:teuthology.orchestra.run.smithi040:> /usr/sbin/getenforce 2023-07-14T10:42:15.412 INFO:teuthology.orchestra.run.smithi040.stdout:Permissive 2023-07-14T10:42:15.413 DEBUG:teuthology.orchestra.run.smithi045:> /usr/sbin/getenforce 2023-07-14T10:42:15.439 INFO:teuthology.orchestra.run.smithi045.stdout:Permissive 2023-07-14T10:42:15.440 DEBUG:teuthology.orchestra.run.smithi130:> /usr/sbin/getenforce 2023-07-14T10:42:15.463 INFO:teuthology.orchestra.run.smithi130.stdout:Permissive 2023-07-14T10:42:15.464 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi040.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi045.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi130.front.sepia.ceph.com': 'permissive'} 2023-07-14T10:42:15.464 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T10:42:15.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:42:15.536 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T10:42:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:42:15.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T10:42:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T10:42:15.595 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-07-14T10:42:15.595 DEBUG:teuthology.orchestra.run.smithi040:> sudo /usr/sbin/setenforce permissive 2023-07-14T10:42:15.665 DEBUG:teuthology.orchestra.run.smithi045:> sudo /usr/sbin/setenforce permissive 2023-07-14T10:42:15.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo /usr/sbin/setenforce permissive 2023-07-14T10:42:15.720 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-07-14T10:42:15.730 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-07-14T10:42:15.731 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-07-14T10:42:15.752 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-07-14T10:42:15.754 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi040.front.sepia.ceph.com,smithi045.front.sepia.ceph.com,smithi130.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-07-14T10:55:49.766 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi040.front.sepia.ceph.com'), Remote(name='ubuntu@smithi045.front.sepia.ceph.com'), Remote(name='ubuntu@smithi130.front.sepia.ceph.com')] 2023-07-14T10:55:49.777 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:55:49.806 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:55:49.947 DEBUG:teuthology.orchestra.run.smithi040:> true 2023-07-14T10:55:50.060 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-07-14T10:55:50.101 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2023-07-14T10:55:50.101 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:55:50.252 DEBUG:teuthology.orchestra.run.smithi045:> true 2023-07-14T10:55:50.330 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi045.front.sepia.ceph.com' 2023-07-14T10:55:50.332 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-07-14T10:55:50.333 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-07-14T10:55:50.408 DEBUG:teuthology.orchestra.run.smithi130:> true 2023-07-14T10:55:50.480 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2023-07-14T10:55:50.481 INFO:teuthology.run_tasks:Running task clock... 2023-07-14T10:55:50.492 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-07-14T10:55:50.493 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-07-14T10:55:50.494 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T10:55:50.508 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-07-14T10:55:50.508 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T10:55:50.511 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-07-14T10:55:50.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T10:55:50.548 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-07-14T10:55:50.564 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-07-14T10:55:50.564 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-07-14T10:55:50.577 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-07-14T10:55:50.577 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-07-14T10:55:50.613 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-07-14T10:55:50.638 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: ntpd: command not found 2023-07-14T10:55:50.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: ntpd: command not found 2023-07-14T10:55:50.654 INFO:teuthology.orchestra.run.smithi045.stdout:506 Cannot talk to daemon 2023-07-14T10:55:50.665 INFO:teuthology.orchestra.run.smithi130.stdout:506 Cannot talk to daemon 2023-07-14T10:55:50.669 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-07-14T10:55:50.686 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-07-14T10:55:50.686 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-07-14T10:55:50.705 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-07-14T10:55:50.706 INFO:teuthology.orchestra.run.smithi040.stderr:sudo: ntpd: command not found 2023-07-14T10:55:50.749 INFO:teuthology.orchestra.run.smithi040.stdout:506 Cannot talk to daemon 2023-07-14T10:55:50.766 INFO:teuthology.orchestra.run.smithi045.stderr:bash: ntpq: command not found 2023-07-14T10:55:50.770 INFO:teuthology.orchestra.run.smithi130.stderr:bash: ntpq: command not found 2023-07-14T10:55:50.771 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T10:55:50.771 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2023-07-14T10:55:50.772 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.772 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.772 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.772 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.772 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-07-14T10:55:50.778 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T10:55:50.779 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2023-07-14T10:55:50.779 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.779 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.779 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.779 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.816 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-07-14T10:55:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:bash: ntpq: command not found 2023-07-14T10:55:50.914 INFO:teuthology.orchestra.run.smithi040.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T10:55:50.914 INFO:teuthology.orchestra.run.smithi040.stdout:=============================================================================== 2023-07-14T10:55:50.914 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.936 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.936 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.936 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T10:55:50.942 INFO:teuthology.run_tasks:Running task pexec... 2023-07-14T10:55:50.999 INFO:teuthology.task.pexec:Executing custom commands... 2023-07-14T10:55:51.000 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T10:55:51.001 DEBUG:teuthology.orchestra.run.smithi040:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-07-14T10:55:51.010 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi045.front.sepia.ceph.com 2023-07-14T10:55:51.011 DEBUG:teuthology.orchestra.run.smithi045:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-07-14T10:55:51.013 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi130.front.sepia.ceph.com 2023-07-14T10:55:51.013 DEBUG:teuthology.orchestra.run.smithi130:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-07-14T10:55:51.388 INFO:teuthology.orchestra.run.smithi130.stdout:Updating Subscription Management repositories. 2023-07-14T10:55:51.412 INFO:teuthology.orchestra.run.smithi045.stdout:Updating Subscription Management repositories. 2023-07-14T10:55:51.534 INFO:teuthology.orchestra.run.smithi040.stdout:Updating Subscription Management repositories. 2023-07-14T10:55:53.169 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:05:44 ago on Fri 14 Jul 2023 10:50:08 AM UTC. 2023-07-14T10:55:53.197 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:05:19 ago on Fri 14 Jul 2023 10:50:33 AM UTC. 2023-07-14T10:55:53.508 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:17 ago on Fri 14 Jul 2023 10:53:35 AM UTC. 2023-07-14T10:55:54.880 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout:Resetting modules: 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout: container-tools 2023-07-14T10:55:54.881 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:55:54.882 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2023-07-14T10:55:54.882 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2023-07-14T10:55:54.882 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:55:55.111 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2023-07-14T10:55:55.115 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2023-07-14T10:55:55.115 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout:Resetting modules: 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout: container-tools 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2023-07-14T10:55:55.116 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:55:55.575 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-07-14T10:55:55.578 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-07-14T10:55:55.578 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-07-14T10:55:55.579 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-07-14T10:55:55.579 INFO:teuthology.orchestra.run.smithi040.stdout:Resetting modules: 2023-07-14T10:55:55.579 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools 2023-07-14T10:55:55.579 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:55:55.580 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-07-14T10:55:55.580 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-07-14T10:55:55.580 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:55:55.876 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:05:47 ago on Fri 14 Jul 2023 10:50:08 AM UTC. 2023-07-14T10:55:56.109 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:05:22 ago on Fri 14 Jul 2023 10:50:33 AM UTC. 2023-07-14T10:55:56.874 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:21 ago on Fri 14 Jul 2023 10:53:35 AM UTC. 2023-07-14T10:56:09.283 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2023-07-14T10:56:09.404 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2023-07-14T10:56:09.778 INFO:teuthology.orchestra.run.smithi045.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:09.873 INFO:teuthology.orchestra.run.smithi130.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:10.340 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-07-14T10:56:10.992 INFO:teuthology.orchestra.run.smithi040.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:11.432 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:05:37 ago on Fri 14 Jul 2023 10:50:33 AM UTC. 2023-07-14T10:56:11.465 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:06:03 ago on Fri 14 Jul 2023 10:50:08 AM UTC. 2023-07-14T10:56:12.880 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:37 ago on Fri 14 Jul 2023 10:53:35 AM UTC. 2023-07-14T10:56:13.178 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2023-07-14T10:56:13.272 INFO:teuthology.orchestra.run.smithi130.stdout:============================================================================================================ 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout:============================================================================================================ 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout:Installing group/module packages: 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-07-14T10:56:13.273 INFO:teuthology.orchestra.run.smithi130.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-07-14T10:56:13.274 INFO:teuthology.orchestra.run.smithi130.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-07-14T10:56:13.275 INFO:teuthology.orchestra.run.smithi130.stdout:Installing module profiles: 2023-07-14T10:56:13.280 INFO:teuthology.orchestra.run.smithi130.stdout: container-tools/common 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout:Enabling module streams: 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout: container-tools rhel8 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout:============================================================================================================ 2023-07-14T10:56:13.281 INFO:teuthology.orchestra.run.smithi130.stdout:Install 10 Packages 2023-07-14T10:56:13.282 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:56:13.282 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 18 M 2023-07-14T10:56:13.282 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 65 M 2023-07-14T10:56:13.282 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2023-07-14T10:56:13.376 INFO:teuthology.orchestra.run.smithi130.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 402 kB/s | 41 kB 00:00 2023-07-14T10:56:13.397 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2023-07-14T10:56:13.416 INFO:teuthology.orchestra.run.smithi130.stdout:(2/10): python3-pyxdg-0.25-16.el8.noarch.rpm 654 kB/s | 94 kB 00:00 2023-07-14T10:56:13.490 INFO:teuthology.orchestra.run.smithi045.stdout:============================================================================================================ 2023-07-14T10:56:13.490 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout:============================================================================================================ 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout:Installing group/module packages: 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-07-14T10:56:13.491 INFO:teuthology.orchestra.run.smithi045.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-07-14T10:56:13.492 INFO:teuthology.orchestra.run.smithi045.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout:Installing module profiles: 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout: container-tools/common 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout:Enabling module streams: 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout: container-tools rhel8 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2023-07-14T10:56:13.493 INFO:teuthology.orchestra.run.smithi045.stdout:============================================================================================================ 2023-07-14T10:56:13.494 INFO:teuthology.orchestra.run.smithi045.stdout:Install 10 Packages 2023-07-14T10:56:13.494 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:56:13.495 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 18 M 2023-07-14T10:56:13.496 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 65 M 2023-07-14T10:56:13.496 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2023-07-14T10:56:13.497 INFO:teuthology.orchestra.run.smithi130.stdout:(3/10): python3-pytoml-0.1.14-5.git7dea353.el8. 110 kB/s | 25 kB 00:00 2023-07-14T10:56:13.540 INFO:teuthology.orchestra.run.smithi130.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 311 kB/s | 49 kB 00:00 2023-07-14T10:56:13.575 INFO:teuthology.orchestra.run.smithi130.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 3.2 MB/s | 524 kB 00:00 2023-07-14T10:56:13.649 INFO:teuthology.orchestra.run.smithi130.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 1.2 MB/s | 149 kB 00:00 2023-07-14T10:56:13.691 INFO:teuthology.orchestra.run.smithi045.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 207 kB/s | 41 kB 00:00 2023-07-14T10:56:13.724 INFO:teuthology.orchestra.run.smithi045.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 107 kB/s | 25 kB 00:00 2023-07-14T10:56:13.755 INFO:teuthology.orchestra.run.smithi045.stdout:(3/10): python3-pyxdg-0.25-16.el8.noarch.rpm 366 kB/s | 94 kB 00:00 2023-07-14T10:56:13.792 INFO:teuthology.orchestra.run.smithi130.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 982 kB/s | 213 kB 00:00 2023-07-14T10:56:13.811 INFO:teuthology.orchestra.run.smithi045.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 421 kB/s | 49 kB 00:00 2023-07-14T10:56:13.883 INFO:teuthology.orchestra.run.smithi045.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 3.2 MB/s | 524 kB 00:00 2023-07-14T10:56:13.921 INFO:teuthology.orchestra.run.smithi130.stdout:(8/10): buildah-1.26.2-1.module+el8.6.0+16771+2 19 MB/s | 8.0 MB 00:00 2023-07-14T10:56:13.921 INFO:teuthology.orchestra.run.smithi045.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 1.3 MB/s | 149 kB 00:00 2023-07-14T10:56:14.043 INFO:teuthology.orchestra.run.smithi130.stdout:(9/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 8.7 MB/s | 2.2 MB 00:00 2023-07-14T10:56:14.090 INFO:teuthology.orchestra.run.smithi045.stdout:(7/10): buildah-1.26.2-1.module+el8.6.0+16771+2 24 MB/s | 8.0 MB 00:00 2023-07-14T10:56:14.093 INFO:teuthology.orchestra.run.smithi045.stdout:(8/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 1.0 MB/s | 213 kB 00:00 2023-07-14T10:56:14.126 INFO:teuthology.orchestra.run.smithi130.stdout:(10/10): skopeo-1.9.1-1.module+el8.6.0+16771+28 15 MB/s | 7.0 MB 00:00 2023-07-14T10:56:14.135 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:56:14.135 INFO:teuthology.orchestra.run.smithi130.stdout:Total 21 MB/s | 18 MB 00:00 2023-07-14T10:56:14.227 INFO:teuthology.orchestra.run.smithi045.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 23 MB/s | 7.0 MB 00:00 2023-07-14T10:56:14.296 INFO:teuthology.orchestra.run.smithi045.stdout:(10/10): toolbox-0.0.99.3-0.6.module+el8.6.0+16 11 MB/s | 2.2 MB 00:00 2023-07-14T10:56:14.296 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:56:14.305 INFO:teuthology.orchestra.run.smithi045.stdout:Total 23 MB/s | 18 MB 00:00 2023-07-14T10:56:14.331 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2023-07-14T10:56:14.348 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2023-07-14T10:56:14.349 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2023-07-14T10:56:14.493 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2023-07-14T10:56:14.532 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2023-07-14T10:56:14.533 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2023-07-14T10:56:14.547 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2023-07-14T10:56:14.547 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2023-07-14T10:56:14.706 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2023-07-14T10:56:14.712 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2023-07-14T10:56:14.920 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-07-14T10:56:15.021 INFO:teuthology.orchestra.run.smithi040.stdout:============================================================================================================ 2023-07-14T10:56:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout:============================================================================================================ 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout:Installing group/module packages: 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-07-14T10:56:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-07-14T10:56:15.023 INFO:teuthology.orchestra.run.smithi040.stdout:Installing module profiles: 2023-07-14T10:56:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools/common 2023-07-14T10:56:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:56:15.024 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling module streams: 2023-07-14T10:56:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools rhel8 2023-07-14T10:56:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:56:15.030 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-07-14T10:56:15.030 INFO:teuthology.orchestra.run.smithi040.stdout:============================================================================================================ 2023-07-14T10:56:15.030 INFO:teuthology.orchestra.run.smithi040.stdout:Install 10 Packages 2023-07-14T10:56:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:56:15.030 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 18 M 2023-07-14T10:56:15.031 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 65 M 2023-07-14T10:56:15.048 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-07-14T10:56:15.048 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2023-07-14T10:56:15.188 INFO:teuthology.orchestra.run.smithi040.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 567 kB/s | 94 kB 00:00 2023-07-14T10:56:15.212 INFO:teuthology.orchestra.run.smithi040.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 130 kB/s | 25 kB 00:00 2023-07-14T10:56:15.237 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-07-14T10:56:15.240 INFO:teuthology.orchestra.run.smithi040.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 189 kB/s | 41 kB 00:00 2023-07-14T10:56:15.278 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2023-07-14T10:56:15.340 INFO:teuthology.orchestra.run.smithi040.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 328 kB/s | 49 kB 00:00 2023-07-14T10:56:15.413 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:15.415 INFO:teuthology.orchestra.run.smithi040.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 2.7 MB/s | 524 kB 00:00 2023-07-14T10:56:15.453 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-07-14T10:56:15.534 INFO:teuthology.orchestra.run.smithi040.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 778 kB/s | 149 kB 00:00 2023-07-14T10:56:15.642 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-07-14T10:56:15.650 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:15.651 INFO:teuthology.orchestra.run.smithi040.stdout:(7/10): buildah-1.26.2-1.module+el8.6.0+16771+2 19 MB/s | 8.0 MB 00:00 2023-07-14T10:56:15.678 INFO:teuthology.orchestra.run.smithi040.stdout:(8/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 797 kB/s | 213 kB 00:00 2023-07-14T10:56:15.809 INFO:teuthology.orchestra.run.smithi040.stdout:(9/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 14 MB/s | 2.2 MB 00:00 2023-07-14T10:56:15.845 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-07-14T10:56:15.845 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-07-14T10:56:15.914 INFO:teuthology.orchestra.run.smithi040.stdout:(10/10): skopeo-1.9.1-1.module+el8.6.0+16771+28 18 MB/s | 7.0 MB 00:00 2023-07-14T10:56:15.925 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:56:15.925 INFO:teuthology.orchestra.run.smithi040.stdout:Total 20 MB/s | 18 MB 00:00 2023-07-14T10:56:16.051 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-07-14T10:56:16.216 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-07-14T10:56:16.260 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-07-14T10:56:16.280 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-07-14T10:56:16.281 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-07-14T10:56:16.445 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-07-14T10:56:16.785 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-07-14T10:56:16.790 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-07-14T10:56:16.977 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-07-14T10:56:17.236 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-07-14T10:56:17.347 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-07-14T10:56:17.538 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-07-14T10:56:17.744 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:17.892 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-07-14T10:56:17.956 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-07-14T10:56:18.103 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-07-14T10:56:18.177 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-07-14T10:56:18.182 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-07-14T10:56:18.303 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-07-14T10:56:18.471 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:18.534 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-07-14T10:56:18.624 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-07-14T10:56:18.692 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-07-14T10:56:18.743 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:18.744 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-07-14T10:56:18.744 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:18.744 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-07-14T10:56:18.744 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-07-14T10:56:18.744 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-07-14T10:56:18.745 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-07-14T10:56:18.745 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-07-14T10:56:18.746 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-07-14T10:56:18.746 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-07-14T10:56:18.859 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:19.117 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:19.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-07-14T10:56:19.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:19.118 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-07-14T10:56:19.124 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-07-14T10:56:19.125 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-07-14T10:56:19.125 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-07-14T10:56:19.125 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-07-14T10:56:19.125 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-07-14T10:56:19.125 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-07-14T10:56:19.463 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-07-14T10:56:19.872 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-07-14T10:56:19.872 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:06:11 ago on Fri 14 Jul 2023 10:50:08 AM UTC. 2023-07-14T10:56:20.388 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-07-14T10:56:20.388 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:05:46 ago on Fri 14 Jul 2023 10:50:33 AM UTC. 2023-07-14T10:56:20.468 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-07-14T10:56:20.789 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-07-14T10:56:20.984 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-07-14T10:56:21.222 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:21.617 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-07-14T10:56:21.618 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-07-14T10:56:21.618 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-07-14T10:56:21.618 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-07-14T10:56:21.618 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-07-14T10:56:21.618 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-07-14T10:56:21.619 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-07-14T10:56:21.619 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-07-14T10:56:21.620 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-07-14T10:56:21.620 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-07-14T10:56:23.104 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-07-14T10:56:23.104 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:47 ago on Fri 14 Jul 2023 10:53:35 AM UTC. 2023-07-14T10:56:29.507 INFO:teuthology.orchestra.run.smithi045.stdout:Installed products updated. 2023-07-14T10:56:29.749 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:56:29.749 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2023-07-14T10:56:29.749 INFO:teuthology.orchestra.run.smithi045.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:29.749 INFO:teuthology.orchestra.run.smithi045.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:29.750 INFO:teuthology.orchestra.run.smithi045.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:29.751 INFO:teuthology.orchestra.run.smithi045.stdout: yajl-2.1.0-10.el8.x86_64 2023-07-14T10:56:29.751 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:56:29.751 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2023-07-14T10:56:29.906 DEBUG:teuthology.parallel:result is None 2023-07-14T10:56:30.516 INFO:teuthology.orchestra.run.smithi130.stdout:Installed products updated. 2023-07-14T10:56:30.737 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:56:30.737 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2023-07-14T10:56:30.737 INFO:teuthology.orchestra.run.smithi130.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:30.737 INFO:teuthology.orchestra.run.smithi130.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:30.738 INFO:teuthology.orchestra.run.smithi130.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:30.738 INFO:teuthology.orchestra.run.smithi130.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-07-14T10:56:30.738 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-07-14T10:56:30.738 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-07-14T10:56:30.744 INFO:teuthology.orchestra.run.smithi130.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:30.744 INFO:teuthology.orchestra.run.smithi130.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:30.745 INFO:teuthology.orchestra.run.smithi130.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:30.745 INFO:teuthology.orchestra.run.smithi130.stdout: yajl-2.1.0-10.el8.x86_64 2023-07-14T10:56:30.745 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:56:30.745 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2023-07-14T10:56:30.893 DEBUG:teuthology.parallel:result is None 2023-07-14T10:56:33.862 INFO:teuthology.orchestra.run.smithi040.stdout:Installed products updated. 2023-07-14T10:56:34.076 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:56:34.077 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-07-14T10:56:34.077 INFO:teuthology.orchestra.run.smithi040.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:34.077 INFO:teuthology.orchestra.run.smithi040.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:34.077 INFO:teuthology.orchestra.run.smithi040.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:34.077 INFO:teuthology.orchestra.run.smithi040.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: yajl-2.1.0-10.el8.x86_64 2023-07-14T10:56:34.078 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:56:34.079 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-07-14T10:56:34.301 DEBUG:teuthology.parallel:result is None 2023-07-14T10:56:34.302 INFO:teuthology.run_tasks:Running task install... 2023-07-14T10:56:34.325 DEBUG:teuthology.task.install:project ceph 2023-07-14T10:56:34.326 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b'}} 2023-07-14T10:56:34.326 DEBUG:teuthology.task.install:config {'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'flavor': 'default', 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b'} 2023-07-14T10:56:34.327 INFO:teuthology.task.install:Using flavor: default 2023-07-14T10:56:34.341 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-07-14T10:56:34.342 INFO:teuthology.task.install:extra packages: {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']} 2023-07-14T10:56:34.342 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b', 'tag': None, 'wait_for_package': False} 2023-07-14T10:56:34.343 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T10:56:34.346 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b', 'tag': None, 'wait_for_package': False} 2023-07-14T10:56:34.346 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T10:56:34.348 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': {'deb': ['python3-cephfs', 'cephfs-shell', 'cephfs-top', 'cephfs-mirror'], 'rpm': ['python3-cephfs', 'cephfs-top', 'cephfs-mirror']}, 'extra_system_packages': {'deb': ['bison', 'flex', 'libelf-dev', 'libssl-dev', 'network-manager', 'iproute2', 'util-linux', 'dump', 'indent', 'libaio-dev', 'libtool-bin', 'uuid-dev', 'xfslibs-dev', 'postgresql', 'postgresql-client', 'postgresql-common', 'postgresql-contrib'], 'rpm': ['bison', 'flex', 'elfutils-libelf-devel', 'openssl-devel', 'NetworkManager', 'iproute', 'util-linux', 'libacl-devel', 'libaio-devel', 'libattr-devel', 'libtool', 'libuuid-devel', 'xfsdump', 'xfsprogs', 'xfsprogs-devel', 'libaio-devel', 'libtool', 'libuuid-devel', 'xfsprogs-devel', 'postgresql', 'postgresql-server', 'postgresql-contrib']}, 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b', 'tag': None, 'wait_for_package': False} 2023-07-14T10:56:34.355 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T10:56:34.685 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/ 2023-07-14T10:56:34.686 INFO:teuthology.task.install.rpm:Package version is 18.0.0-4757.g3fc0089c 2023-07-14T10:56:34.696 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/ 2023-07-14T10:56:34.696 INFO:teuthology.task.install.rpm:Package version is 18.0.0-4757.g3fc0089c 2023-07-14T10:56:34.698 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/ 2023-07-14T10:56:34.699 INFO:teuthology.task.install.rpm:Package version is 18.0.0-4757.g3fc0089c 2023-07-14T10:56:34.890 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-07-14T10:56:34.890 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T10:56:34.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-07-14T10:56:34.907 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-07-14T10:56:34.907 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T10:56:34.907 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-07-14T10:56:34.932 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-07-14T10:56:34.932 DEBUG:teuthology.orchestra.run.smithi130:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/3fc0089cc1082313a98853d48b892acdc1385b8b/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-07-14T10:56:34.933 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-07-14T10:56:34.933 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T10:56:34.934 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-07-14T10:56:34.965 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-07-14T10:56:34.965 DEBUG:teuthology.orchestra.run.smithi045:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/3fc0089cc1082313a98853d48b892acdc1385b8b/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-07-14T10:56:35.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-07-14T10:56:35.010 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-cephfs, cephfs-top, cephfs-mirror, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib, bison, flex, elfutils-libelf-devel, openssl-devel, NetworkManager, iproute, util-linux, libacl-devel, libaio-devel, libattr-devel, libtool, libuuid-devel, xfsdump, xfsprogs, xfsprogs-devel, libaio-devel, libtool, libuuid-devel, xfsprogs-devel, postgresql, postgresql-server, postgresql-contrib on remote rpm x86_64 2023-07-14T10:56:35.010 DEBUG:teuthology.orchestra.run.smithi040:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/3fc0089cc1082313a98853d48b892acdc1385b8b/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-07-14T10:56:35.043 DEBUG:teuthology.orchestra.run.smithi045:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-07-14T10:56:35.097 DEBUG:teuthology.orchestra.run.smithi130:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-07-14T10:56:35.134 DEBUG:teuthology.orchestra.run.smithi045:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-07-14T10:56:35.151 DEBUG:teuthology.orchestra.run.smithi040:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-07-14T10:56:35.196 INFO:teuthology.orchestra.run.smithi130.stdout:check_obsoletes = 1 2023-07-14T10:56:35.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum clean all 2023-07-14T10:56:35.206 INFO:teuthology.orchestra.run.smithi045.stdout:check_obsoletes = 1 2023-07-14T10:56:35.207 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum clean all 2023-07-14T10:56:35.250 DEBUG:teuthology.orchestra.run.smithi040:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-07-14T10:56:35.333 INFO:teuthology.orchestra.run.smithi040.stdout:check_obsoletes = 1 2023-07-14T10:56:35.337 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum clean all 2023-07-14T10:56:35.572 INFO:teuthology.orchestra.run.smithi130.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:35.627 INFO:teuthology.orchestra.run.smithi045.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:35.824 INFO:teuthology.orchestra.run.smithi040.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:36.715 INFO:teuthology.orchestra.run.smithi130.stdout:44 files removed 2023-07-14T10:56:36.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-07-14T10:56:36.892 INFO:teuthology.orchestra.run.smithi045.stdout:44 files removed 2023-07-14T10:56:36.947 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-07-14T10:56:37.103 INFO:teuthology.orchestra.run.smithi130.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:37.114 INFO:teuthology.orchestra.run.smithi040.stdout:44 files removed 2023-07-14T10:56:37.181 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-cephfs cephfs-top cephfs-mirror bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib bison flex elfutils-libelf-devel openssl-devel NetworkManager iproute util-linux libacl-devel libaio-devel libattr-devel libtool libuuid-devel xfsdump xfsprogs xfsprogs-devel libaio-devel libtool libuuid-devel xfsprogs-devel postgresql postgresql-server postgresql-contrib 2023-07-14T10:56:37.318 INFO:teuthology.orchestra.run.smithi045.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:37.667 INFO:teuthology.orchestra.run.smithi040.stdout:Updating Subscription Management repositories. 2023-07-14T10:56:38.651 INFO:teuthology.orchestra.run.smithi130.stdout:ceph packages for x86_64 195 kB/s | 81 kB 00:00 2023-07-14T10:56:38.909 INFO:teuthology.orchestra.run.smithi045.stdout:ceph packages for x86_64 196 kB/s | 81 kB 00:00 2023-07-14T10:56:39.039 INFO:teuthology.orchestra.run.smithi130.stdout:ceph noarch packages 26 kB/s | 9.4 kB 00:00 2023-07-14T10:56:39.312 INFO:teuthology.orchestra.run.smithi045.stdout:ceph noarch packages 26 kB/s | 9.4 kB 00:00 2023-07-14T10:56:39.399 INFO:teuthology.orchestra.run.smithi040.stdout:ceph packages for x86_64 196 kB/s | 81 kB 00:00 2023-07-14T10:56:39.430 INFO:teuthology.orchestra.run.smithi130.stdout:ceph source packages 4.0 kB/s | 1.5 kB 00:00 2023-07-14T10:56:39.590 INFO:teuthology.orchestra.run.smithi130.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-07-14T10:56:39.712 INFO:teuthology.orchestra.run.smithi045.stdout:ceph source packages 4.2 kB/s | 1.5 kB 00:00 2023-07-14T10:56:39.800 INFO:teuthology.orchestra.run.smithi040.stdout:ceph noarch packages 26 kB/s | 9.4 kB 00:00 2023-07-14T10:56:39.944 INFO:teuthology.orchestra.run.smithi045.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2023-07-14T10:56:40.175 INFO:teuthology.orchestra.run.smithi040.stdout:ceph source packages 4.2 kB/s | 1.5 kB 00:00 2023-07-14T10:56:40.348 INFO:teuthology.orchestra.run.smithi040.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-07-14T10:56:40.460 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2023-07-14T10:56:41.125 INFO:teuthology.orchestra.run.smithi045.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-07-14T10:56:42.156 INFO:teuthology.orchestra.run.smithi040.stdout:Extra Packages for Enterprise Linux 8.7 MB/s | 16 MB 00:01 2023-07-14T10:56:43.949 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 333 kB/s | 24 kB 00:00 2023-07-14T10:56:44.819 INFO:teuthology.orchestra.run.smithi045.stdout:lab-extras 469 kB/s | 24 kB 00:00 2023-07-14T10:56:45.157 INFO:teuthology.orchestra.run.smithi130.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 47 MB/s | 53 MB 00:01 2023-07-14T10:56:46.053 INFO:teuthology.orchestra.run.smithi045.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 41 MB/s | 47 MB 00:01 2023-07-14T10:56:46.396 INFO:teuthology.orchestra.run.smithi040.stdout:lab-extras 430 kB/s | 24 kB 00:00 2023-07-14T10:56:47.970 INFO:teuthology.orchestra.run.smithi040.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 35 MB/s | 53 MB 00:01 2023-07-14T10:56:53.449 INFO:teuthology.orchestra.run.smithi130.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 45 MB/s | 47 MB 00:01 2023-07-14T10:56:54.702 INFO:teuthology.orchestra.run.smithi045.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 49 MB/s | 53 MB 00:01 2023-07-14T10:56:57.681 INFO:teuthology.orchestra.run.smithi040.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 31 MB/s | 47 MB 00:01 2023-07-14T10:57:04.605 INFO:teuthology.orchestra.run.smithi130.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:04.608 INFO:teuthology.orchestra.run.smithi130.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:04.625 INFO:teuthology.orchestra.run.smithi130.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:04.625 INFO:teuthology.orchestra.run.smithi130.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:04.625 INFO:teuthology.orchestra.run.smithi130.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-07-14T10:57:04.625 INFO:teuthology.orchestra.run.smithi130.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-07-14T10:57:04.637 INFO:teuthology.orchestra.run.smithi130.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-07-14T10:57:04.637 INFO:teuthology.orchestra.run.smithi130.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-07-14T10:57:04.638 INFO:teuthology.orchestra.run.smithi130.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-07-14T10:57:04.762 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2023-07-14T10:57:04.772 INFO:teuthology.orchestra.run.smithi130.stdout:=========================================================================================================================== 2023-07-14T10:57:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2023-07-14T10:57:04.773 INFO:teuthology.orchestra.run.smithi130.stdout:=========================================================================================================================== 2023-07-14T10:57:04.773 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2023-07-14T10:57:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-07-14T10:57:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-07-14T10:57:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: ceph x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.4 k 2023-07-14T10:57:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 5.4 M 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 848 k 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 165 k 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 1.5 M 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 129 k 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 1.7 M 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 7.4 M 2023-07-14T10:57:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 47 k 2023-07-14T10:57:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 13 M 2023-07-14T10:57:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 48 M 2023-07-14T10:57:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-volume noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 258 k 2023-07-14T10:57:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 166 k 2023-07-14T10:57:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 258 k 2023-07-14T10:57:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-top noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 18 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 31 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 771 k 2023-07-14T10:57:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 139 k 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-07-14T10:57:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 197 k 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 371 k 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 364 k 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 107 k 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 94 k 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.4 M 2023-07-14T10:57:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 179 k 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrading: 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: librados2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.7 M 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.6 M 2023-07-14T10:57:04.799 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 22 M 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 23 k 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 2.3 M 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 240 k 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 4.6 M 2023-07-14T10:57:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 18 M 2023-07-14T10:57:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 15 k 2023-07-14T10:57:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 24 k 2023-07-14T10:57:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-07-14T10:57:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 137 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-07-14T10:57:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-07-14T10:57:04.818 INFO:teuthology.orchestra.run.smithi130.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-07-14T10:57:04.818 INFO:teuthology.orchestra.run.smithi130.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-07-14T10:57:04.818 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-07-14T10:57:04.818 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-07-14T10:57:04.821 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 473 k 2023-07-14T10:57:04.821 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.7 M 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-07-14T10:57:04.822 INFO:teuthology.orchestra.run.smithi130.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-07-14T10:57:04.823 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-07-14T10:57:04.824 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-07-14T10:57:04.824 INFO:teuthology.orchestra.run.smithi130.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 45 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 119 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-07-14T10:57:04.826 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-07-14T10:57:04.827 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-07-14T10:57:04.828 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-07-14T10:57:04.829 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-07-14T10:57:04.830 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:04.831 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-07-14T10:57:04.831 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-07-14T10:57:04.831 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-07-14T10:57:04.831 INFO:teuthology.orchestra.run.smithi130.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-07-14T10:57:04.837 INFO:teuthology.orchestra.run.smithi130.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-07-14T10:57:04.837 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-07-14T10:57:04.837 INFO:teuthology.orchestra.run.smithi130.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-07-14T10:57:04.837 INFO:teuthology.orchestra.run.smithi130.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-07-14T10:57:04.838 INFO:teuthology.orchestra.run.smithi130.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout:Enabling module streams: 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql 10 2023-07-14T10:57:04.839 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:57:04.840 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2023-07-14T10:57:04.840 INFO:teuthology.orchestra.run.smithi130.stdout:=========================================================================================================================== 2023-07-14T10:57:04.841 INFO:teuthology.orchestra.run.smithi130.stdout:Install 128 Packages 2023-07-14T10:57:04.841 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrade 3 Packages 2023-07-14T10:57:04.841 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T10:57:04.842 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 205 M 2023-07-14T10:57:04.842 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2023-07-14T10:57:05.144 INFO:teuthology.orchestra.run.smithi130.stdout:(1/131): ceph-18.0.0-4757.g3fc0089c.el8.x86_64. 34 kB/s | 6.4 kB 00:00 2023-07-14T10:57:05.494 INFO:teuthology.orchestra.run.smithi130.stdout:(2/131): ceph-fuse-18.0.0-4757.g3fc0089c.el8.x8 2.4 MB/s | 848 kB 00:00 2023-07-14T10:57:05.596 INFO:teuthology.orchestra.run.smithi130.stdout:(3/131): ceph-immutable-object-cache-18.0.0-475 1.6 MB/s | 165 kB 00:00 2023-07-14T10:57:05.695 INFO:teuthology.orchestra.run.smithi130.stdout:(4/131): ceph-base-18.0.0-4757.g3fc0089c.el8.x8 7.3 MB/s | 5.4 MB 00:00 2023-07-14T10:57:05.931 INFO:teuthology.orchestra.run.smithi130.stdout:(5/131): ceph-mgr-18.0.0-4757.g3fc0089c.el8.x86 6.7 MB/s | 1.5 MB 00:00 2023-07-14T10:57:05.997 INFO:teuthology.orchestra.run.smithi130.stdout:(6/131): ceph-mds-18.0.0-4757.g3fc0089c.el8.x86 5.7 MB/s | 2.3 MB 00:00 2023-07-14T10:57:06.510 INFO:teuthology.orchestra.run.smithi130.stdout:(7/131): ceph-mon-18.0.0-4757.g3fc0089c.el8.x86 8.1 MB/s | 4.6 MB 00:00 2023-07-14T10:57:06.700 INFO:teuthology.orchestra.run.smithi045.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:06.701 INFO:teuthology.orchestra.run.smithi045.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:06.714 INFO:teuthology.orchestra.run.smithi045.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:06.714 INFO:teuthology.orchestra.run.smithi045.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:06.721 INFO:teuthology.orchestra.run.smithi045.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-07-14T10:57:06.721 INFO:teuthology.orchestra.run.smithi045.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-07-14T10:57:06.721 INFO:teuthology.orchestra.run.smithi045.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-07-14T10:57:06.723 INFO:teuthology.orchestra.run.smithi045.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-07-14T10:57:06.723 INFO:teuthology.orchestra.run.smithi045.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-07-14T10:57:06.869 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2023-07-14T10:57:06.884 INFO:teuthology.orchestra.run.smithi045.stdout:=========================================================================================================================== 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout: Package Arch Version Repository Size 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout:=========================================================================================================================== 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout: ceph x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.4 k 2023-07-14T10:57:06.885 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-base x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 5.4 M 2023-07-14T10:57:06.886 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 848 k 2023-07-14T10:57:06.886 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 165 k 2023-07-14T10:57:06.886 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 1.5 M 2023-07-14T10:57:06.886 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 129 k 2023-07-14T10:57:06.887 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-dashboard noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 1.7 M 2023-07-14T10:57:06.888 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 7.4 M 2023-07-14T10:57:06.888 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-rook noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 47 k 2023-07-14T10:57:06.889 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-radosgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 13 M 2023-07-14T10:57:06.897 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-test x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 48 M 2023-07-14T10:57:06.897 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-volume noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 258 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 166 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: cephfs-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 258 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: cephfs-top noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 18 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-07-14T10:57:06.898 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 31 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 771 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: librados-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 139 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-07-14T10:57:06.899 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-07-14T10:57:06.900 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cephfs x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 197 k 2023-07-14T10:57:06.900 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rados x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 371 k 2023-07-14T10:57:06.901 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 364 k 2023-07-14T10:57:06.902 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 107 k 2023-07-14T10:57:06.902 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 94 k 2023-07-14T10:57:06.903 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.4 M 2023-07-14T10:57:06.903 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-nbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 179 k 2023-07-14T10:57:06.903 INFO:teuthology.orchestra.run.smithi045.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-07-14T10:57:06.903 INFO:teuthology.orchestra.run.smithi045.stdout:Upgrading: 2023-07-14T10:57:06.903 INFO:teuthology.orchestra.run.smithi045.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-07-14T10:57:06.904 INFO:teuthology.orchestra.run.smithi045.stdout: librados2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.7 M 2023-07-14T10:57:06.904 INFO:teuthology.orchestra.run.smithi045.stdout: librbd1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.6 M 2023-07-14T10:57:06.904 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2023-07-14T10:57:06.904 INFO:teuthology.orchestra.run.smithi045.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-07-14T10:57:06.913 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 22 M 2023-07-14T10:57:06.913 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-grafana-dashboards noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 23 k 2023-07-14T10:57:06.913 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mds x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 2.3 M 2023-07-14T10:57:06.913 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-modules-core noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 240 k 2023-07-14T10:57:06.914 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mon x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 4.6 M 2023-07-14T10:57:06.914 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-osd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 18 M 2023-07-14T10:57:06.914 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-prometheus-alerts noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 15 k 2023-07-14T10:57:06.914 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-selinux x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 24 k 2023-07-14T10:57:06.914 INFO:teuthology.orchestra.run.smithi045.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-07-14T10:57:06.917 INFO:teuthology.orchestra.run.smithi045.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: libcephsqlite x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 137 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-07-14T10:57:06.918 INFO:teuthology.orchestra.run.smithi045.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: libradosstriper1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 473 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: librgw2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.7 M 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-07-14T10:57:06.919 INFO:teuthology.orchestra.run.smithi045.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-07-14T10:57:06.920 INFO:teuthology.orchestra.run.smithi045.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-07-14T10:57:06.925 INFO:teuthology.orchestra.run.smithi045.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-07-14T10:57:06.925 INFO:teuthology.orchestra.run.smithi045.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-07-14T10:57:06.925 INFO:teuthology.orchestra.run.smithi045.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-07-14T10:57:06.925 INFO:teuthology.orchestra.run.smithi045.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-07-14T10:57:06.926 INFO:teuthology.orchestra.run.smithi045.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-argparse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 45 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 119 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-07-14T10:57:06.927 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-07-14T10:57:06.928 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-07-14T10:57:06.928 INFO:teuthology.orchestra.run.smithi045.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-07-14T10:57:06.928 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-07-14T10:57:06.936 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-07-14T10:57:06.937 INFO:teuthology.orchestra.run.smithi045.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-07-14T10:57:06.938 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-07-14T10:57:06.939 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-07-14T10:57:06.941 INFO:teuthology.orchestra.run.smithi045.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-07-14T10:57:06.941 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:06.941 INFO:teuthology.orchestra.run.smithi045.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-07-14T10:57:06.941 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-07-14T10:57:06.942 INFO:teuthology.orchestra.run.smithi045.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-07-14T10:57:06.943 INFO:teuthology.orchestra.run.smithi045.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-07-14T10:57:06.944 INFO:teuthology.orchestra.run.smithi045.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout:Enabling module streams: 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql 10 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2023-07-14T10:57:06.945 INFO:teuthology.orchestra.run.smithi045.stdout:=========================================================================================================================== 2023-07-14T10:57:06.946 INFO:teuthology.orchestra.run.smithi045.stdout:Install 128 Packages 2023-07-14T10:57:06.946 INFO:teuthology.orchestra.run.smithi045.stdout:Upgrade 3 Packages 2023-07-14T10:57:06.946 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:57:06.947 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 205 M 2023-07-14T10:57:06.947 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2023-07-14T10:57:07.124 INFO:teuthology.orchestra.run.smithi130.stdout:(8/131): ceph-common-18.0.0-4757.g3fc0089c.el8. 10 MB/s | 22 MB 00:02 2023-07-14T10:57:07.222 INFO:teuthology.orchestra.run.smithi130.stdout:(9/131): ceph-selinux-18.0.0-4757.g3fc0089c.el8 243 kB/s | 24 kB 00:00 2023-07-14T10:57:07.281 INFO:teuthology.orchestra.run.smithi045.stdout:(1/131): ceph-18.0.0-4757.g3fc0089c.el8.x86_64. 31 kB/s | 6.4 kB 00:00 2023-07-14T10:57:07.747 INFO:teuthology.orchestra.run.smithi130.stdout:(10/131): ceph-radosgw-18.0.0-4757.g3fc0089c.el 11 MB/s | 13 MB 00:01 2023-07-14T10:57:07.848 INFO:teuthology.orchestra.run.smithi130.stdout:(11/131): cephfs-mirror-18.0.0-4757.g3fc0089c.e 2.5 MB/s | 258 kB 00:00 2023-07-14T10:57:07.940 INFO:teuthology.orchestra.run.smithi130.stdout:(12/131): libcephfs-devel-18.0.0-4757.g3fc0089c 338 kB/s | 31 kB 00:00 2023-07-14T10:57:08.088 INFO:teuthology.orchestra.run.smithi130.stdout:(13/131): libcephfs2-18.0.0-4757.g3fc0089c.el8. 5.3 MB/s | 771 kB 00:00 2023-07-14T10:57:08.184 INFO:teuthology.orchestra.run.smithi130.stdout:(14/131): libcephsqlite-18.0.0-4757.g3fc0089c.e 1.3 MB/s | 137 kB 00:00 2023-07-14T10:57:08.282 INFO:teuthology.orchestra.run.smithi130.stdout:(15/131): librados-devel-18.0.0-4757.g3fc0089c. 1.4 MB/s | 139 kB 00:00 2023-07-14T10:57:08.400 INFO:teuthology.orchestra.run.smithi130.stdout:(16/131): libradosstriper1-18.0.0-4757.g3fc0089 4.0 MB/s | 473 kB 00:00 2023-07-14T10:57:08.480 INFO:teuthology.orchestra.run.smithi045.stdout:(2/131): ceph-fuse-18.0.0-4757.g3fc0089c.el8.x8 707 kB/s | 848 kB 00:01 2023-07-14T10:57:08.692 INFO:teuthology.orchestra.run.smithi045.stdout:(3/131): ceph-immutable-object-cache-18.0.0-475 794 kB/s | 165 kB 00:00 2023-07-14T10:57:08.866 INFO:teuthology.orchestra.run.smithi130.stdout:(17/131): ceph-osd-18.0.0-4757.g3fc0089c.el8.x8 6.2 MB/s | 18 MB 00:02 2023-07-14T10:57:08.967 INFO:teuthology.orchestra.run.smithi130.stdout:(18/131): python3-ceph-argparse-18.0.0-4757.g3f 448 kB/s | 45 kB 00:00 2023-07-14T10:57:09.068 INFO:teuthology.orchestra.run.smithi130.stdout:(19/131): python3-ceph-common-18.0.0-4757.g3fc0 1.2 MB/s | 119 kB 00:00 2023-07-14T10:57:09.177 INFO:teuthology.orchestra.run.smithi130.stdout:(20/131): python3-cephfs-18.0.0-4757.g3fc0089c. 1.9 MB/s | 197 kB 00:00 2023-07-14T10:57:09.266 INFO:teuthology.orchestra.run.smithi130.stdout:(21/131): librgw2-18.0.0-4757.g3fc0089c.el8.x86 7.8 MB/s | 6.7 MB 00:00 2023-07-14T10:57:09.301 INFO:teuthology.orchestra.run.smithi130.stdout:(22/131): python3-rados-18.0.0-4757.g3fc0089c.e 2.7 MB/s | 371 kB 00:00 2023-07-14T10:57:09.343 INFO:teuthology.orchestra.run.smithi130.stdout:(23/131): python3-rbd-18.0.0-4757.g3fc0089c.el8 4.3 MB/s | 364 kB 00:00 2023-07-14T10:57:09.397 INFO:teuthology.orchestra.run.smithi130.stdout:(24/131): python3-rgw-18.0.0-4757.g3fc0089c.el8 1.1 MB/s | 107 kB 00:00 2023-07-14T10:57:09.430 INFO:teuthology.orchestra.run.smithi130.stdout:(25/131): rbd-fuse-18.0.0-4757.g3fc0089c.el8.x8 1.1 MB/s | 94 kB 00:00 2023-07-14T10:57:09.536 INFO:teuthology.orchestra.run.smithi130.stdout:(26/131): rbd-nbd-18.0.0-4757.g3fc0089c.el8.x86 1.6 MB/s | 179 kB 00:00 2023-07-14T10:57:09.634 INFO:teuthology.orchestra.run.smithi130.stdout:(27/131): ceph-grafana-dashboards-18.0.0-4757.g 251 kB/s | 23 kB 00:00 2023-07-14T10:57:09.738 INFO:teuthology.orchestra.run.smithi130.stdout:(28/131): ceph-mgr-cephadm-18.0.0-4757.g3fc0089 1.3 MB/s | 129 kB 00:00 2023-07-14T10:57:09.764 INFO:teuthology.orchestra.run.smithi045.stdout:(4/131): ceph-mds-18.0.0-4757.g3fc0089c.el8.x86 2.1 MB/s | 2.3 MB 00:01 2023-07-14T10:57:09.908 INFO:teuthology.orchestra.run.smithi130.stdout:(29/131): rbd-mirror-18.0.0-4757.g3fc0089c.el8. 6.8 MB/s | 3.4 MB 00:00 2023-07-14T10:57:10.039 INFO:teuthology.orchestra.run.smithi130.stdout:(30/131): ceph-mgr-dashboard-18.0.0-4757.g3fc00 5.5 MB/s | 1.7 MB 00:00 2023-07-14T10:57:10.149 INFO:teuthology.orchestra.run.smithi130.stdout:(31/131): ceph-mgr-modules-core-18.0.0-4757.g3f 2.2 MB/s | 240 kB 00:00 2023-07-14T10:57:10.150 INFO:teuthology.orchestra.run.smithi040.stdout:Package librados2-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:10.150 INFO:teuthology.orchestra.run.smithi040.stdout:Package librbd1-1:12.2.7-9.el8.x86_64 is already installed. 2023-07-14T10:57:10.158 INFO:teuthology.orchestra.run.smithi040.stdout:Package NetworkManager-1:1.36.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:10.159 INFO:teuthology.orchestra.run.smithi040.stdout:Package iproute-5.15.0-4.el8.x86_64 is already installed. 2023-07-14T10:57:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Package util-linux-2.32.1-35.el8.x86_64 is already installed. 2023-07-14T10:57:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Package libacl-devel-2.2.53-1.el8.x86_64 is already installed. 2023-07-14T10:57:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Package libattr-devel-2.4.48-3.el8.x86_64 is already installed. 2023-07-14T10:57:10.179 INFO:teuthology.orchestra.run.smithi040.stdout:Package xfsdump-3.1.8-4.el8.x86_64 is already installed. 2023-07-14T10:57:10.179 INFO:teuthology.orchestra.run.smithi040.stdout:Package xfsprogs-5.0.0-10.el8.x86_64 is already installed. 2023-07-14T10:57:10.234 INFO:teuthology.orchestra.run.smithi130.stdout:(32/131): ceph-mgr-rook-18.0.0-4757.g3fc0089c.e 568 kB/s | 47 kB 00:00 2023-07-14T10:57:10.317 INFO:teuthology.orchestra.run.smithi045.stdout:(5/131): ceph-mgr-18.0.0-4757.g3fc0089c.el8.x86 3.0 MB/s | 1.5 MB 00:00 2023-07-14T10:57:10.317 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-07-14T10:57:10.318 INFO:teuthology.orchestra.run.smithi130.stdout:(33/131): ceph-prometheus-alerts-18.0.0-4757.g3 193 kB/s | 15 kB 00:00 2023-07-14T10:57:10.332 INFO:teuthology.orchestra.run.smithi040.stdout:=========================================================================================================================== 2023-07-14T10:57:10.332 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-07-14T10:57:10.332 INFO:teuthology.orchestra.run.smithi040.stdout:=========================================================================================================================== 2023-07-14T10:57:10.332 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: NetworkManager-initscripts-updown noarch 1:1.36.0-9.el8_6 rhel-8-for-x86_64-baseos-rpms 138 k 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: bison x86_64 3.0.4-10.el8 rhel-8-for-x86_64-appstream-rpms 688 k 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: ceph x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.4 k 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-base x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 5.4 M 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 848 k 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-immutable-object-cache x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 165 k 2023-07-14T10:57:10.333 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 1.5 M 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 129 k 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-dashboard noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 1.7 M 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-diskprediction-local noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 7.4 M 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-rook noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 47 k 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-radosgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 13 M 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-test x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 48 M 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-volume noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 258 k 2023-07-14T10:57:10.334 INFO:teuthology.orchestra.run.smithi040.stdout: cephadm noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 166 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: cephfs-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 258 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: cephfs-top noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 18 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: elfutils-libelf-devel x86_64 0.186-1.el8 rhel-8-for-x86_64-baseos-rpms 60 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: flex x86_64 2.6.1-9.el8 rhel-8-for-x86_64-appstream-rpms 320 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: libaio-devel x86_64 0.3.112-1.el8 rhel-8-for-x86_64-baseos-rpms 19 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 31 k 2023-07-14T10:57:10.335 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 771 k 2023-07-14T10:57:10.336 INFO:teuthology.orchestra.run.smithi040.stdout: librados-devel x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 139 k 2023-07-14T10:57:10.336 INFO:teuthology.orchestra.run.smithi040.stdout: libtool x86_64 2.4.6-25.el8 rhel-8-for-x86_64-appstream-rpms 709 k 2023-07-14T10:57:10.336 INFO:teuthology.orchestra.run.smithi040.stdout: libuuid-devel x86_64 2.32.1-35.el8 rhel-8-for-x86_64-baseos-rpms 98 k 2023-07-14T10:57:10.351 INFO:teuthology.orchestra.run.smithi040.stdout: openssl-devel x86_64 1:1.1.1k-7.el8_6 rhel-8-for-x86_64-baseos-rpms 2.3 M 2023-07-14T10:57:10.351 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 1.5 M 2023-07-14T10:57:10.360 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql-contrib x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 808 k 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql-server x86_64 10.21-2.module+el8.6.0+15342+53518fac rhel-8-for-x86_64-appstream-rpms 5.1 M 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cephfs x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 197 k 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rados x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 371 k 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 364 k 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rgw x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 107 k 2023-07-14T10:57:10.361 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-fuse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 94 k 2023-07-14T10:57:10.362 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-mirror x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.4 M 2023-07-14T10:57:10.362 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-nbd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 179 k 2023-07-14T10:57:10.362 INFO:teuthology.orchestra.run.smithi040.stdout: xfsprogs-devel x86_64 5.0.0-10.el8 rhel-8-for-x86_64-baseos-rpms 72 k 2023-07-14T10:57:10.362 INFO:teuthology.orchestra.run.smithi040.stdout:Upgrading: 2023-07-14T10:57:10.362 INFO:teuthology.orchestra.run.smithi040.stdout: iproute x86_64 5.15.0-4.el8_6.1 rhel-8-for-x86_64-baseos-rpms 798 k 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout: librados2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.7 M 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout: librbd1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 3.6 M 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout: automake noarch 1.16.1-7.el8 rhel-8-for-x86_64-appstream-rpms 713 k 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 22 M 2023-07-14T10:57:10.363 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-grafana-dashboards noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 23 k 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mds x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 2.3 M 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-modules-core noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 240 k 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mon x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 4.6 M 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-osd x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 18 M 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-prometheus-alerts noarch 2:18.0.0-4757.g3fc0089c.el8 ceph-noarch 15 k 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-selinux x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 24 k 2023-07-14T10:57:10.364 INFO:teuthology.orchestra.run.smithi040.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: keyutils-libs-devel x86_64 1.5.10-9.el8 rhel-8-for-x86_64-baseos-rpms 48 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: krb5-devel x86_64 1.18.2-14.el8 rhel-8-for-x86_64-baseos-rpms 560 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: libcephsqlite x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 137 k 2023-07-14T10:57:10.365 INFO:teuthology.orchestra.run.smithi040.stdout: libcom_err-devel x86_64 1.45.6-4.el8 rhel-8-for-x86_64-baseos-rpms 39 k 2023-07-14T10:57:10.369 INFO:teuthology.orchestra.run.smithi040.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-07-14T10:57:10.370 INFO:teuthology.orchestra.run.smithi040.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-07-14T10:57:10.370 INFO:teuthology.orchestra.run.smithi040.stdout: libpq x86_64 13.5-1.el8 rhel-8-for-x86_64-appstream-rpms 198 k 2023-07-14T10:57:10.370 INFO:teuthology.orchestra.run.smithi040.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: libradosstriper1 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 473 k 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: librgw2 x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 6.7 M 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: libselinux-devel x86_64 2.9-5.el8 rhel-8-for-x86_64-baseos-rpms 200 k 2023-07-14T10:57:10.371 INFO:teuthology.orchestra.run.smithi040.stdout: libsepol-devel x86_64 2.9-3.el8 rhel-8-for-x86_64-baseos-rpms 87 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: libverto-devel x86_64 0.3.0-5.el8 rhel-8-for-x86_64-baseos-rpms 18 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-devel x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 605 k 2023-07-14T10:57:10.372 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-utf16 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 229 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-utf32 x86_64 10.32-2.el8 rhel-8-for-x86_64-baseos-rpms 220 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-07-14T10:57:10.373 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-argparse x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 45 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-common x86_64 2:18.0.0-4757.g3fc0089c.el8 ceph 119 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-07-14T10:57:10.374 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-07-14T10:57:10.375 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-07-14T10:57:10.375 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-07-14T10:57:10.375 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-07-14T10:57:10.375 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-07-14T10:57:10.375 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:10.376 INFO:teuthology.orchestra.run.smithi040.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-07-14T10:57:10.377 INFO:teuthology.orchestra.run.smithi040.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-07-14T10:57:10.377 INFO:teuthology.orchestra.run.smithi040.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-07-14T10:57:10.377 INFO:teuthology.orchestra.run.smithi040.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-07-14T10:57:10.378 INFO:teuthology.orchestra.run.smithi040.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-07-14T10:57:10.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-07-14T10:57:10.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-07-14T10:57:10.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-07-14T10:57:10.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-07-14T10:57:10.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-07-14T10:57:10.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-07-14T10:57:10.392 INFO:teuthology.orchestra.run.smithi040.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-07-14T10:57:10.392 INFO:teuthology.orchestra.run.smithi040.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-07-14T10:57:10.392 INFO:teuthology.orchestra.run.smithi040.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-07-14T10:57:10.393 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rsa noarch 4.8-1.el8 epel 61 k 2023-07-14T10:57:10.393 INFO:teuthology.orchestra.run.smithi040.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-07-14T10:57:10.393 INFO:teuthology.orchestra.run.smithi040.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-07-14T10:57:10.393 INFO:teuthology.orchestra.run.smithi040.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-07-14T10:57:10.393 INFO:teuthology.orchestra.run.smithi040.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-07-14T10:57:10.398 INFO:teuthology.orchestra.run.smithi040.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-07-14T10:57:10.399 INFO:teuthology.orchestra.run.smithi040.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-07-14T10:57:10.399 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-07-14T10:57:10.400 INFO:teuthology.orchestra.run.smithi040.stdout: uuid x86_64 1.6.2-43.el8 rhel-8-for-x86_64-appstream-rpms 64 k 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout: zlib-devel x86_64 1.2.11-18.el8_5 rhel-8-for-x86_64-baseos-rpms 58 k 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling module streams: 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql 10 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:57:10.401 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-07-14T10:57:10.402 INFO:teuthology.orchestra.run.smithi040.stdout:=========================================================================================================================== 2023-07-14T10:57:10.402 INFO:teuthology.orchestra.run.smithi040.stdout:Install 128 Packages 2023-07-14T10:57:10.402 INFO:teuthology.orchestra.run.smithi040.stdout:Upgrade 3 Packages 2023-07-14T10:57:10.402 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:57:10.403 INFO:teuthology.orchestra.run.smithi130.stdout:(34/131): ceph-volume-18.0.0-4757.g3fc0089c.el8 2.8 MB/s | 258 kB 00:00 2023-07-14T10:57:10.412 INFO:teuthology.orchestra.run.smithi045.stdout:(6/131): ceph-base-18.0.0-4757.g3fc0089c.el8.x8 1.6 MB/s | 5.4 MB 00:03 2023-07-14T10:57:10.412 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 205 M 2023-07-14T10:57:10.412 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-07-14T10:57:10.507 INFO:teuthology.orchestra.run.smithi130.stdout:(35/131): cephadm-18.0.0-4757.g3fc0089c.el8.noa 1.5 MB/s | 166 kB 00:00 2023-07-14T10:57:10.607 INFO:teuthology.orchestra.run.smithi130.stdout:(36/131): cephfs-top-18.0.0-4757.g3fc0089c.el8. 179 kB/s | 18 kB 00:00 2023-07-14T10:57:10.864 INFO:teuthology.orchestra.run.smithi130.stdout:(37/131): ceph-mgr-diskprediction-local-18.0.0- 7.7 MB/s | 7.4 MB 00:00 2023-07-14T10:57:11.023 INFO:teuthology.orchestra.run.smithi040.stdout:(1/131): ceph-18.0.0-4757.g3fc0089c.el8.x86_64. 24 kB/s | 6.4 kB 00:00 2023-07-14T10:57:11.115 INFO:teuthology.orchestra.run.smithi045.stdout:(7/131): ceph-mon-18.0.0-4757.g3fc0089c.el8.x86 5.5 MB/s | 4.6 MB 00:00 2023-07-14T10:57:11.323 INFO:teuthology.orchestra.run.smithi040.stdout:(2/131): ceph-fuse-18.0.0-4757.g3fc0089c.el8.x8 2.8 MB/s | 848 kB 00:00 2023-07-14T10:57:11.390 INFO:teuthology.orchestra.run.smithi040.stdout:(3/131): ceph-immutable-object-cache-18.0.0-475 2.4 MB/s | 165 kB 00:00 2023-07-14T10:57:11.467 INFO:teuthology.orchestra.run.smithi130.stdout:(38/131): ceph-test-18.0.0-4757.g3fc0089c.el8.x 11 MB/s | 48 MB 00:04 2023-07-14T10:57:11.641 INFO:teuthology.orchestra.run.smithi040.stdout:(4/131): ceph-mds-18.0.0-4757.g3fc0089c.el8.x86 9.2 MB/s | 2.3 MB 00:00 2023-07-14T10:57:11.825 INFO:teuthology.orchestra.run.smithi040.stdout:(5/131): ceph-mgr-18.0.0-4757.g3fc0089c.el8.x86 8.5 MB/s | 1.5 MB 00:00 2023-07-14T10:57:12.326 INFO:teuthology.orchestra.run.smithi040.stdout:(6/131): ceph-mon-18.0.0-4757.g3fc0089c.el8.x86 9.2 MB/s | 4.6 MB 00:00 2023-07-14T10:57:12.415 INFO:teuthology.orchestra.run.smithi045.stdout:(8/131): ceph-radosgw-18.0.0-4757.g3fc0089c.el8 10 MB/s | 13 MB 00:01 2023-07-14T10:57:12.497 INFO:teuthology.orchestra.run.smithi045.stdout:(9/131): ceph-selinux-18.0.0-4757.g3fc0089c.el8 326 kB/s | 24 kB 00:00 2023-07-14T10:57:14.011 INFO:teuthology.orchestra.run.smithi040.stdout:(7/131): ceph-osd-18.0.0-4757.g3fc0089c.el8.x86 11 MB/s | 18 MB 00:01 2023-07-14T10:57:14.102 INFO:teuthology.orchestra.run.smithi040.stdout:(8/131): ceph-base-18.0.0-4757.g3fc0089c.el8.x8 1.6 MB/s | 5.4 MB 00:03 2023-07-14T10:57:14.201 INFO:teuthology.orchestra.run.smithi040.stdout:(9/131): ceph-selinux-18.0.0-4757.g3fc0089c.el8 243 kB/s | 24 kB 00:00 2023-07-14T10:57:15.296 INFO:teuthology.orchestra.run.smithi045.stdout:(10/131): ceph-common-18.0.0-4757.g3fc0089c.el8 2.7 MB/s | 22 MB 00:08 2023-07-14T10:57:15.557 INFO:teuthology.orchestra.run.smithi045.stdout:(11/131): cephfs-mirror-18.0.0-4757.g3fc0089c.e 969 kB/s | 258 kB 00:00 2023-07-14T10:57:15.666 INFO:teuthology.orchestra.run.smithi045.stdout:(12/131): libcephfs-devel-18.0.0-4757.g3fc0089c 286 kB/s | 31 kB 00:00 2023-07-14T10:57:15.767 INFO:teuthology.orchestra.run.smithi040.stdout:(10/131): ceph-radosgw-18.0.0-4757.g3fc0089c.el 7.7 MB/s | 13 MB 00:01 2023-07-14T10:57:15.876 INFO:teuthology.orchestra.run.smithi040.stdout:(11/131): cephfs-mirror-18.0.0-4757.g3fc0089c.e 2.3 MB/s | 258 kB 00:00 2023-07-14T10:57:15.992 INFO:teuthology.orchestra.run.smithi040.stdout:(12/131): libcephfs-devel-18.0.0-4757.g3fc0089c 266 kB/s | 31 kB 00:00 2023-07-14T10:57:16.028 INFO:teuthology.orchestra.run.smithi045.stdout:(13/131): libcephfs2-18.0.0-4757.g3fc0089c.el8. 2.1 MB/s | 771 kB 00:00 2023-07-14T10:57:16.483 INFO:teuthology.orchestra.run.smithi040.stdout:(13/131): ceph-common-18.0.0-4757.g3fc0089c.el8 3.9 MB/s | 22 MB 00:05 2023-07-14T10:57:16.584 INFO:teuthology.orchestra.run.smithi040.stdout:(14/131): libcephsqlite-18.0.0-4757.g3fc0089c.e 1.2 MB/s | 137 kB 00:00 2023-07-14T10:57:16.680 INFO:teuthology.orchestra.run.smithi040.stdout:(15/131): librados-devel-18.0.0-4757.g3fc0089c. 1.5 MB/s | 139 kB 00:00 2023-07-14T10:57:16.703 INFO:teuthology.orchestra.run.smithi045.stdout:(14/131): libcephsqlite-18.0.0-4757.g3fc0089c.e 206 kB/s | 137 kB 00:00 2023-07-14T10:57:16.793 INFO:teuthology.orchestra.run.smithi040.stdout:(16/131): libcephfs2-18.0.0-4757.g3fc0089c.el8. 973 kB/s | 771 kB 00:00 2023-07-14T10:57:16.803 INFO:teuthology.orchestra.run.smithi045.stdout:(15/131): librados-devel-18.0.0-4757.g3fc0089c. 1.3 MB/s | 139 kB 00:00 2023-07-14T10:57:16.860 INFO:teuthology.orchestra.run.smithi040.stdout:(17/131): libradosstriper1-18.0.0-4757.g3fc0089 2.5 MB/s | 473 kB 00:00 2023-07-14T10:57:16.945 INFO:teuthology.orchestra.run.smithi040.stdout:(18/131): python3-ceph-argparse-18.0.0-4757.g3f 536 kB/s | 45 kB 00:00 2023-07-14T10:57:17.038 INFO:teuthology.orchestra.run.smithi040.stdout:(19/131): python3-ceph-common-18.0.0-4757.g3fc0 1.3 MB/s | 119 kB 00:00 2023-07-14T10:57:17.038 INFO:teuthology.orchestra.run.smithi045.stdout:(16/131): libradosstriper1-18.0.0-4757.g3fc0089 2.0 MB/s | 473 kB 00:00 2023-07-14T10:57:17.153 INFO:teuthology.orchestra.run.smithi040.stdout:(20/131): python3-cephfs-18.0.0-4757.g3fc0089c. 1.6 MB/s | 197 kB 00:00 2023-07-14T10:57:17.595 INFO:teuthology.orchestra.run.smithi040.stdout:(21/131): python3-rados-18.0.0-4757.g3fc0089c.e 840 kB/s | 371 kB 00:00 2023-07-14T10:57:17.711 INFO:teuthology.orchestra.run.smithi040.stdout:(22/131): python3-rbd-18.0.0-4757.g3fc0089c.el8 3.3 MB/s | 364 kB 00:00 2023-07-14T10:57:17.847 INFO:teuthology.orchestra.run.smithi040.stdout:(23/131): librgw2-18.0.0-4757.g3fc0089c.el8.x86 6.4 MB/s | 6.7 MB 00:01 2023-07-14T10:57:17.878 INFO:teuthology.orchestra.run.smithi040.stdout:(24/131): python3-rgw-18.0.0-4757.g3fc0089c.el8 640 kB/s | 107 kB 00:00 2023-07-14T10:57:17.946 INFO:teuthology.orchestra.run.smithi040.stdout:(25/131): rbd-fuse-18.0.0-4757.g3fc0089c.el8.x8 947 kB/s | 94 kB 00:00 2023-07-14T10:57:18.082 INFO:teuthology.orchestra.run.smithi040.stdout:(26/131): rbd-nbd-18.0.0-4757.g3fc0089c.el8.x86 1.4 MB/s | 179 kB 00:00 2023-07-14T10:57:18.179 INFO:teuthology.orchestra.run.smithi040.stdout:(27/131): ceph-grafana-dashboards-18.0.0-4757.g 231 kB/s | 23 kB 00:00 2023-07-14T10:57:18.289 INFO:teuthology.orchestra.run.smithi040.stdout:(28/131): ceph-mgr-cephadm-18.0.0-4757.g3fc0089 1.1 MB/s | 129 kB 00:00 2023-07-14T10:57:18.347 INFO:teuthology.orchestra.run.smithi040.stdout:(29/131): rbd-mirror-18.0.0-4757.g3fc0089c.el8. 7.2 MB/s | 3.4 MB 00:00 2023-07-14T10:57:18.573 INFO:teuthology.orchestra.run.smithi040.stdout:(30/131): ceph-mgr-dashboard-18.0.0-4757.g3fc00 6.0 MB/s | 1.7 MB 00:00 2023-07-14T10:57:18.692 INFO:teuthology.orchestra.run.smithi040.stdout:(31/131): ceph-mgr-modules-core-18.0.0-4757.g3f 2.2 MB/s | 240 kB 00:00 2023-07-14T10:57:18.735 INFO:teuthology.orchestra.run.smithi045.stdout:(17/131): ceph-test-18.0.0-4757.g3fc0089c.el8.x 7.7 MB/s | 48 MB 00:06 2023-07-14T10:57:18.760 INFO:teuthology.orchestra.run.smithi040.stdout:(32/131): ceph-mgr-rook-18.0.0-4757.g3fc0089c.e 632 kB/s | 47 kB 00:00 2023-07-14T10:57:18.836 INFO:teuthology.orchestra.run.smithi045.stdout:(18/131): python3-ceph-argparse-18.0.0-4757.g3f 449 kB/s | 45 kB 00:00 2023-07-14T10:57:18.851 INFO:teuthology.orchestra.run.smithi040.stdout:(33/131): ceph-prometheus-alerts-18.0.0-4757.g3 159 kB/s | 15 kB 00:00 2023-07-14T10:57:18.921 INFO:teuthology.orchestra.run.smithi045.stdout:(19/131): python3-ceph-common-18.0.0-4757.g3fc0 1.4 MB/s | 119 kB 00:00 2023-07-14T10:57:18.949 INFO:teuthology.orchestra.run.smithi040.stdout:(34/131): ceph-volume-18.0.0-4757.g3fc0089c.el8 2.5 MB/s | 258 kB 00:00 2023-07-14T10:57:19.029 INFO:teuthology.orchestra.run.smithi045.stdout:(20/131): python3-cephfs-18.0.0-4757.g3fc0089c. 1.8 MB/s | 197 kB 00:00 2023-07-14T10:57:19.057 INFO:teuthology.orchestra.run.smithi040.stdout:(35/131): cephadm-18.0.0-4757.g3fc0089c.el8.noa 1.5 MB/s | 166 kB 00:00 2023-07-14T10:57:19.119 INFO:teuthology.orchestra.run.smithi045.stdout:(21/131): python3-rados-18.0.0-4757.g3fc0089c.e 4.0 MB/s | 371 kB 00:00 2023-07-14T10:57:19.175 INFO:teuthology.orchestra.run.smithi040.stdout:(36/131): ceph-mgr-diskprediction-local-18.0.0- 9.0 MB/s | 7.4 MB 00:00 2023-07-14T10:57:19.202 INFO:teuthology.orchestra.run.smithi045.stdout:(22/131): python3-rbd-18.0.0-4757.g3fc0089c.el8 4.3 MB/s | 364 kB 00:00 2023-07-14T10:57:19.202 INFO:teuthology.orchestra.run.smithi040.stdout:(37/131): cephfs-top-18.0.0-4757.g3fc0089c.el8. 126 kB/s | 18 kB 00:00 2023-07-14T10:57:19.291 INFO:teuthology.orchestra.run.smithi045.stdout:(23/131): python3-rgw-18.0.0-4757.g3fc0089c.el8 1.2 MB/s | 107 kB 00:00 2023-07-14T10:57:19.370 INFO:teuthology.orchestra.run.smithi045.stdout:(24/131): rbd-fuse-18.0.0-4757.g3fc0089c.el8.x8 1.2 MB/s | 94 kB 00:00 2023-07-14T10:57:19.516 INFO:teuthology.orchestra.run.smithi040.stdout:(38/131): liboath-2.6.2-3.el8.x86_64.rpm 191 kB/s | 59 kB 00:00 2023-07-14T10:57:19.584 INFO:teuthology.orchestra.run.smithi040.stdout:(39/131): gperftools-libs-2.7-9.el8.x86_64.rpm 749 kB/s | 306 kB 00:00 2023-07-14T10:57:19.618 INFO:teuthology.orchestra.run.smithi040.stdout:(40/131): libunwind-1.3.1-3.el8.x86_64.rpm 750 kB/s | 75 kB 00:00 2023-07-14T10:57:19.685 INFO:teuthology.orchestra.run.smithi040.stdout:(41/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 582 kB/s | 44 kB 00:00 2023-07-14T10:57:19.716 INFO:teuthology.orchestra.run.smithi045.stdout:(25/131): librgw2-18.0.0-4757.g3fc0089c.el8.x86 2.5 MB/s | 6.7 MB 00:02 2023-07-14T10:57:19.717 INFO:teuthology.orchestra.run.smithi040.stdout:(42/131): python3-asyncssh-2.7.0-2.el8.noarch.r 3.5 MB/s | 476 kB 00:00 2023-07-14T10:57:19.793 INFO:teuthology.orchestra.run.smithi040.stdout:(43/131): python3-cachetools-3.1.1-4.el8.noarch 439 kB/s | 33 kB 00:00 2023-07-14T10:57:19.805 INFO:teuthology.orchestra.run.smithi045.stdout:(26/131): rbd-mirror-18.0.0-4757.g3fc0089c.el8. 7.8 MB/s | 3.4 MB 00:00 2023-07-14T10:57:19.824 INFO:teuthology.orchestra.run.smithi040.stdout:(44/131): python3-beautifulsoup4-4.6.3-2.el8.1. 1.4 MB/s | 185 kB 00:00 2023-07-14T10:57:19.831 INFO:teuthology.orchestra.run.smithi045.stdout:(27/131): rbd-nbd-18.0.0-4757.g3fc0089c.el8.x86 1.5 MB/s | 179 kB 00:00 2023-07-14T10:57:19.861 INFO:teuthology.orchestra.run.smithi040.stdout:(45/131): python3-certifi-2018.10.15-7.el8.noar 242 kB/s | 16 kB 00:00 2023-07-14T10:57:19.862 INFO:teuthology.orchestra.run.smithi045.stdout:(28/131): ceph-grafana-dashboards-18.0.0-4757.g 396 kB/s | 23 kB 00:00 2023-07-14T10:57:19.897 INFO:teuthology.orchestra.run.smithi040.stdout:(46/131): python3-cheroot-8.5.2-1.el8.noarch.rp 2.2 MB/s | 173 kB 00:00 2023-07-14T10:57:19.904 INFO:teuthology.orchestra.run.smithi045.stdout:(29/131): ceph-mgr-cephadm-18.0.0-4757.g3fc0089 1.7 MB/s | 129 kB 00:00 2023-07-14T10:57:19.962 INFO:teuthology.orchestra.run.smithi040.stdout:(47/131): python3-google-auth-1.1.1-10.el8.noar 1.2 MB/s | 81 kB 00:00 2023-07-14T10:57:19.987 INFO:teuthology.orchestra.run.smithi040.stdout:(48/131): python3-cherrypy-18.4.0-1.el8.noarch. 3.0 MB/s | 384 kB 00:00 2023-07-14T10:57:20.064 INFO:teuthology.orchestra.run.smithi045.stdout:(30/131): ceph-mgr-dashboard-18.0.0-4757.g3fc00 8.6 MB/s | 1.7 MB 00:00 2023-07-14T10:57:20.148 INFO:teuthology.orchestra.run.smithi045.stdout:(31/131): ceph-mgr-modules-core-18.0.0-4757.g3f 2.8 MB/s | 240 kB 00:00 2023-07-14T10:57:20.213 INFO:teuthology.orchestra.run.smithi045.stdout:(32/131): ceph-mgr-rook-18.0.0-4757.g3fc0089c.e 714 kB/s | 47 kB 00:00 2023-07-14T10:57:20.280 INFO:teuthology.orchestra.run.smithi045.stdout:(33/131): ceph-prometheus-alerts-18.0.0-4757.g3 218 kB/s | 15 kB 00:00 2023-07-14T10:57:20.349 INFO:teuthology.orchestra.run.smithi045.stdout:(34/131): ceph-volume-18.0.0-4757.g3fc0089c.el8 3.8 MB/s | 258 kB 00:00 2023-07-14T10:57:20.415 INFO:teuthology.orchestra.run.smithi045.stdout:(35/131): cephadm-18.0.0-4757.g3fc0089c.el8.noa 2.4 MB/s | 166 kB 00:00 2023-07-14T10:57:20.470 INFO:teuthology.orchestra.run.smithi040.stdout:(49/131): ceph-test-18.0.0-4757.g3fc0089c.el8.x 7.6 MB/s | 48 MB 00:06 2023-07-14T10:57:20.473 INFO:teuthology.orchestra.run.smithi045.stdout:(36/131): cephfs-top-18.0.0-4757.g3fc0089c.el8. 308 kB/s | 18 kB 00:00 2023-07-14T10:57:20.491 INFO:teuthology.orchestra.run.smithi040.stdout:(50/131): python3-jaraco-6.2-6.el8.noarch.rpm 21 kB/s | 11 kB 00:00 2023-07-14T10:57:20.530 INFO:teuthology.orchestra.run.smithi040.stdout:(51/131): python3-jaraco-functools-2.0-4.el8.no 33 kB/s | 18 kB 00:00 2023-07-14T10:57:20.556 INFO:teuthology.orchestra.run.smithi040.stdout:(52/131): python3-logutils-0.3.5-11.el8.noarch. 729 kB/s | 49 kB 00:00 2023-07-14T10:57:20.731 INFO:teuthology.orchestra.run.smithi045.stdout:(37/131): gperftools-libs-2.7-9.el8.x86_64.rpm 1.2 MB/s | 306 kB 00:00 2023-07-14T10:57:20.791 INFO:teuthology.orchestra.run.smithi045.stdout:(38/131): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-07-14T10:57:20.834 INFO:teuthology.orchestra.run.smithi040.stdout:(53/131): python3-natsort-7.1.1-2.el8.noarch.rp 219 kB/s | 60 kB 00:00 2023-07-14T10:57:20.848 INFO:teuthology.orchestra.run.smithi045.stdout:(39/131): libunwind-1.3.1-3.el8.x86_64.rpm 1.3 MB/s | 75 kB 00:00 2023-07-14T10:57:20.915 INFO:teuthology.orchestra.run.smithi040.stdout:(54/131): python3-kubernetes-11.0.0-6.el8.noarc 2.6 MB/s | 1.2 MB 00:00 2023-07-14T10:57:20.967 INFO:teuthology.orchestra.run.smithi045.stdout:(40/131): python3-asyncssh-2.7.0-2.el8.noarch.r 4.0 MB/s | 476 kB 00:00 2023-07-14T10:57:20.980 INFO:teuthology.orchestra.run.smithi040.stdout:(55/131): python3-pecan-1.3.2-9.el8.noarch.rpm 1.8 MB/s | 283 kB 00:00 2023-07-14T10:57:20.997 INFO:teuthology.orchestra.run.smithi040.stdout:(56/131): python3-portend-2.6-1.el8.noarch.rpm 187 kB/s | 16 kB 00:00 2023-07-14T10:57:21.036 INFO:teuthology.orchestra.run.smithi045.stdout:(41/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 656 kB/s | 44 kB 00:00 2023-07-14T10:57:21.064 INFO:teuthology.orchestra.run.smithi040.stdout:(57/131): python3-more-itertools-7.2.0-3.el8.no 107 kB/s | 59 kB 00:00 2023-07-14T10:57:21.083 INFO:teuthology.orchestra.run.smithi040.stdout:(58/131): python3-repoze-lru-0.7-6.el8.noarch.r 334 kB/s | 34 kB 00:00 2023-07-14T10:57:21.092 INFO:teuthology.orchestra.run.smithi045.stdout:(42/131): python3-beautifulsoup4-4.6.3-2.el8.1. 3.1 MB/s | 185 kB 00:00 2023-07-14T10:57:21.119 INFO:teuthology.orchestra.run.smithi040.stdout:(59/131): python3-routes-2.4.1-12.el8.noarch.rp 1.8 MB/s | 196 kB 00:00 2023-07-14T10:57:21.142 INFO:teuthology.orchestra.run.smithi040.stdout:(60/131): python3-simplegeneric-0.8.1-17.el8.no 321 kB/s | 19 kB 00:00 2023-07-14T10:57:21.162 INFO:teuthology.orchestra.run.smithi045.stdout:(43/131): python3-cachetools-3.1.1-4.el8.noarch 495 kB/s | 33 kB 00:00 2023-07-14T10:57:21.178 INFO:teuthology.orchestra.run.smithi040.stdout:(61/131): python3-singledispatch-3.4.0.3-18.el8 359 kB/s | 24 kB 00:00 2023-07-14T10:57:21.207 INFO:teuthology.orchestra.run.smithi040.stdout:(62/131): python3-tempora-1.14.1-5.el8.noarch.r 437 kB/s | 29 kB 00:00 2023-07-14T10:57:21.217 INFO:teuthology.orchestra.run.smithi045.stdout:(44/131): python3-certifi-2018.10.15-7.el8.noar 279 kB/s | 16 kB 00:00 2023-07-14T10:57:21.233 INFO:teuthology.orchestra.run.smithi040.stdout:(63/131): python3-rsa-4.8-1.el8.noarch.rpm 366 kB/s | 61 kB 00:00 2023-07-14T10:57:21.267 INFO:teuthology.orchestra.run.smithi040.stdout:(64/131): python3-trustme-0.6.0-4.el8.noarch.rp 299 kB/s | 27 kB 00:00 2023-07-14T10:57:21.285 INFO:teuthology.orchestra.run.smithi045.stdout:(45/131): python3-cheroot-8.5.2-1.el8.noarch.rp 2.5 MB/s | 173 kB 00:00 2023-07-14T10:57:21.341 INFO:teuthology.orchestra.run.smithi040.stdout:(65/131): python3-websocket-client-0.56.0-5.el8 919 kB/s | 61 kB 00:00 2023-07-14T10:57:21.366 INFO:teuthology.orchestra.run.smithi040.stdout:(66/131): python3-waitress-1.4.3-1.el8.noarch.r 1.6 MB/s | 255 kB 00:00 2023-07-14T10:57:21.373 INFO:teuthology.orchestra.run.smithi045.stdout:(46/131): python3-cherrypy-18.4.0-1.el8.noarch. 4.5 MB/s | 384 kB 00:00 2023-07-14T10:57:21.434 INFO:teuthology.orchestra.run.smithi040.stdout:(67/131): python3-zc-lockfile-2.0-2.el8.noarch. 345 kB/s | 23 kB 00:00 2023-07-14T10:57:21.435 INFO:teuthology.orchestra.run.smithi045.stdout:(47/131): python3-google-auth-1.1.1-10.el8.noar 1.4 MB/s | 81 kB 00:00 2023-07-14T10:57:21.489 INFO:teuthology.orchestra.run.smithi045.stdout:(48/131): python3-jaraco-6.2-6.el8.noarch.rpm 190 kB/s | 11 kB 00:00 2023-07-14T10:57:21.498 INFO:teuthology.orchestra.run.smithi040.stdout:(68/131): python3-webob-1.8.5-1.el8.1.noarch.rp 969 kB/s | 251 kB 00:00 2023-07-14T10:57:21.552 INFO:teuthology.orchestra.run.smithi040.stdout:(69/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.8 MB/s | 70 kB 00:00 2023-07-14T10:57:21.553 INFO:teuthology.orchestra.run.smithi045.stdout:(49/131): python3-jaraco-functools-2.0-4.el8.no 264 kB/s | 18 kB 00:00 2023-07-14T10:57:21.676 INFO:teuthology.orchestra.run.smithi040.stdout:(70/131): python3-webtest-2.0.33-1.el8.noarch.r 249 kB/s | 85 kB 00:00 2023-07-14T10:57:21.702 INFO:teuthology.orchestra.run.smithi040.stdout:(71/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 255 kB/s | 47 kB 00:00 2023-07-14T10:57:21.727 INFO:teuthology.orchestra.run.smithi045.stdout:(50/131): ceph-osd-18.0.0-4757.g3fc0089c.el8.x8 1.6 MB/s | 18 MB 00:11 2023-07-14T10:57:21.837 INFO:teuthology.orchestra.run.smithi045.stdout:(51/131): ceph-mgr-diskprediction-local-18.0.0- 3.8 MB/s | 7.4 MB 00:01 2023-07-14T10:57:21.857 INFO:teuthology.orchestra.run.smithi040.stdout:(72/131): python3-jwt-1.6.1-2.el8.noarch.rpm 301 kB/s | 43 kB 00:00 2023-07-14T10:57:21.878 INFO:teuthology.orchestra.run.smithi045.stdout:(52/131): python3-kubernetes-11.0.0-6.el8.noarc 3.6 MB/s | 1.2 MB 00:00 2023-07-14T10:57:21.878 INFO:teuthology.orchestra.run.smithi040.stdout:(73/131): libselinux-devel-2.9-5.el8.x86_64.rpm 998 kB/s | 200 kB 00:00 2023-07-14T10:57:21.944 INFO:teuthology.orchestra.run.smithi045.stdout:(53/131): python3-natsort-7.1.1-2.el8.noarch.rp 907 kB/s | 60 kB 00:00 2023-07-14T10:57:21.973 INFO:teuthology.orchestra.run.smithi045.stdout:(54/131): python3-logutils-0.3.5-11.el8.noarch. 201 kB/s | 49 kB 00:00 2023-07-14T10:57:21.981 INFO:teuthology.orchestra.run.smithi040.stdout:(74/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.7 MB/s | 220 kB 00:00 2023-07-14T10:57:22.004 INFO:teuthology.orchestra.run.smithi040.stdout:(75/131): keyutils-libs-devel-1.5.10-9.el8.x86_ 384 kB/s | 48 kB 00:00 2023-07-14T10:57:22.020 INFO:teuthology.orchestra.run.smithi045.stdout:(55/131): python3-pecan-1.3.2-9.el8.noarch.rpm 4.1 MB/s | 283 kB 00:00 2023-07-14T10:57:22.036 INFO:teuthology.orchestra.run.smithi045.stdout:(56/131): python3-portend-2.6-1.el8.noarch.rpm 233 kB/s | 16 kB 00:00 2023-07-14T10:57:22.063 INFO:teuthology.orchestra.run.smithi045.stdout:(57/131): python3-more-itertools-7.2.0-3.el8.no 262 kB/s | 59 kB 00:00 2023-07-14T10:57:22.088 INFO:teuthology.orchestra.run.smithi045.stdout:(58/131): python3-repoze-lru-0.7-6.el8.noarch.r 448 kB/s | 34 kB 00:00 2023-07-14T10:57:22.088 INFO:teuthology.orchestra.run.smithi040.stdout:(76/131): libsepol-devel-2.9-3.el8.x86_64.rpm 803 kB/s | 87 kB 00:00 2023-07-14T10:57:22.106 INFO:teuthology.orchestra.run.smithi040.stdout:(77/131): python3-requests-oauthlib-1.0.0-1.el8 425 kB/s | 43 kB 00:00 2023-07-14T10:57:22.145 INFO:teuthology.orchestra.run.smithi045.stdout:(59/131): python3-rsa-4.8-1.el8.noarch.rpm 819 kB/s | 61 kB 00:00 2023-07-14T10:57:22.164 INFO:teuthology.orchestra.run.smithi045.stdout:(60/131): python3-simplegeneric-0.8.1-17.el8.no 248 kB/s | 19 kB 00:00 2023-07-14T10:57:22.192 INFO:teuthology.orchestra.run.smithi045.stdout:(61/131): python3-routes-2.4.1-12.el8.noarch.rp 1.3 MB/s | 196 kB 00:00 2023-07-14T10:57:22.208 INFO:teuthology.orchestra.run.smithi045.stdout:(62/131): python3-singledispatch-3.4.0.3-18.el8 356 kB/s | 24 kB 00:00 2023-07-14T10:57:22.221 INFO:teuthology.orchestra.run.smithi045.stdout:(63/131): python3-tempora-1.14.1-5.el8.noarch.r 498 kB/s | 29 kB 00:00 2023-07-14T10:57:22.229 INFO:teuthology.orchestra.run.smithi040.stdout:(78/131): python3-pycparser-2.14-14.el8.noarch. 770 kB/s | 109 kB 00:00 2023-07-14T10:57:22.252 INFO:teuthology.orchestra.run.smithi045.stdout:(64/131): python3-trustme-0.6.0-4.el8.noarch.rp 468 kB/s | 27 kB 00:00 2023-07-14T10:57:22.301 INFO:teuthology.orchestra.run.smithi045.stdout:(65/131): python3-webob-1.8.5-1.el8.1.noarch.rp 3.3 MB/s | 251 kB 00:00 2023-07-14T10:57:22.330 INFO:teuthology.orchestra.run.smithi040.stdout:(79/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 1.0 MB/s | 229 kB 00:00 2023-07-14T10:57:22.331 INFO:teuthology.orchestra.run.smithi045.stdout:(66/131): python3-websocket-client-0.56.0-5.el8 814 kB/s | 61 kB 00:00 2023-07-14T10:57:22.374 INFO:teuthology.orchestra.run.smithi045.stdout:(67/131): python3-webtest-2.0.33-1.el8.noarch.r 1.4 MB/s | 85 kB 00:00 2023-07-14T10:57:22.407 INFO:teuthology.orchestra.run.smithi045.stdout:(68/131): python3-waitress-1.4.3-1.el8.noarch.r 1.2 MB/s | 255 kB 00:00 2023-07-14T10:57:22.443 INFO:teuthology.orchestra.run.smithi045.stdout:(69/131): python3-zc-lockfile-2.0-2.el8.noarch. 196 kB/s | 23 kB 00:00 2023-07-14T10:57:22.464 INFO:teuthology.orchestra.run.smithi040.stdout:(80/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 2.3 MB/s | 560 kB 00:00 2023-07-14T10:57:22.465 INFO:teuthology.orchestra.run.smithi045.stdout:(70/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.2 MB/s | 70 kB 00:00 2023-07-14T10:57:22.491 INFO:teuthology.orchestra.run.smithi040.stdout:(81/131): libaio-devel-0.3.112-1.el8.x86_64.rpm 118 kB/s | 19 kB 00:00 2023-07-14T10:57:22.516 INFO:teuthology.orchestra.run.smithi045.stdout:(71/131): thrift-0.13.0-2.el8.x86_64.rpm 11 MB/s | 1.7 MB 00:00 2023-07-14T10:57:22.547 INFO:teuthology.orchestra.run.smithi040.stdout:(82/131): thrift-0.13.0-2.el8.x86_64.rpm 1.6 MB/s | 1.7 MB 00:01 2023-07-14T10:57:22.582 INFO:teuthology.orchestra.run.smithi040.stdout:(83/131): python3-oauthlib-2.1.0-1.el8.noarch.r 1.3 MB/s | 155 kB 00:00 2023-07-14T10:57:22.630 INFO:teuthology.orchestra.run.smithi040.stdout:(84/131): python3-cryptography-3.2.1-5.el8.x86_ 3.8 MB/s | 559 kB 00:00 2023-07-14T10:57:22.699 INFO:teuthology.orchestra.run.smithi045.stdout:(72/131): python3-pyasn1-modules-0.3.7-6.el8.no 489 kB/s | 110 kB 00:00 2023-07-14T10:57:22.732 INFO:teuthology.orchestra.run.smithi045.stdout:(73/131): python3-pytz-2017.2-9.el8.noarch.rpm 260 kB/s | 54 kB 00:00 2023-07-14T10:57:22.797 INFO:teuthology.orchestra.run.smithi040.stdout:(85/131): pcre2-devel-10.32-2.el8.x86_64.rpm 2.8 MB/s | 605 kB 00:00 2023-07-14T10:57:22.825 INFO:teuthology.orchestra.run.smithi040.stdout:(86/131): libverto-devel-0.3.0-5.el8.x86_64.rpm 65 kB/s | 18 kB 00:00 2023-07-14T10:57:22.825 INFO:teuthology.orchestra.run.smithi045.stdout:(74/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.2 MB/s | 259 kB 00:00 2023-07-14T10:57:22.862 INFO:teuthology.orchestra.run.smithi040.stdout:(87/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.0 MB/s | 238 kB 00:00 2023-07-14T10:57:22.933 INFO:teuthology.orchestra.run.smithi045.stdout:(75/131): socat-1.7.4.1-1.el8.x86_64.rpm 1.6 MB/s | 323 kB 00:00 2023-07-14T10:57:22.984 INFO:teuthology.orchestra.run.smithi040.stdout:(88/131): openssl-devel-1.1.1k-7.el8_6.x86_64.r 13 MB/s | 2.3 MB 00:00 2023-07-14T10:57:23.010 INFO:teuthology.orchestra.run.smithi040.stdout:(89/131): xfsprogs-devel-5.0.0-10.el8.x86_64.rp 394 kB/s | 72 kB 00:00 2023-07-14T10:57:23.010 INFO:teuthology.orchestra.run.smithi045.stdout:(76/131): python3-markupsafe-0.23-19.el8.x86_64 208 kB/s | 39 kB 00:00 2023-07-14T10:57:23.032 INFO:teuthology.orchestra.run.smithi040.stdout:(90/131): fuse-2.9.7-15.el8.x86_64.rpm 472 kB/s | 83 kB 00:00 2023-07-14T10:57:23.074 INFO:teuthology.orchestra.run.smithi040.stdout:(91/131): elfutils-libelf-devel-0.186-1.el8.x86 650 kB/s | 60 kB 00:00 2023-07-14T10:57:23.151 INFO:teuthology.orchestra.run.smithi045.stdout:(77/131): python3-scipy-1.0.0-21.module+el8.5.0 19 MB/s | 14 MB 00:00 2023-07-14T10:57:23.159 INFO:teuthology.orchestra.run.smithi040.stdout:(92/131): libuuid-devel-2.32.1-35.el8.x86_64.rp 652 kB/s | 98 kB 00:00 2023-07-14T10:57:23.186 INFO:teuthology.orchestra.run.smithi045.stdout:(78/131): python3-jinja2-2.10.1-3.el8.noarch.rp 2.9 MB/s | 538 kB 00:00 2023-07-14T10:57:23.192 INFO:teuthology.orchestra.run.smithi040.stdout:(93/131): libcom_err-devel-1.45.6-4.el8.x86_64. 331 kB/s | 39 kB 00:00 2023-07-14T10:57:23.211 INFO:teuthology.orchestra.run.smithi045.stdout:(79/131): bison-3.0.4-10.el8.x86_64.rpm 2.4 MB/s | 688 kB 00:00 2023-07-14T10:57:23.216 INFO:teuthology.orchestra.run.smithi040.stdout:(94/131): zlib-devel-1.2.11-18.el8_5.x86_64.rpm 313 kB/s | 58 kB 00:00 2023-07-14T10:57:23.251 INFO:teuthology.orchestra.run.smithi045.stdout:(80/131): python3-mako-1.0.6-13.el8.noarch.rpm 1.5 MB/s | 157 kB 00:00 2023-07-14T10:57:23.374 INFO:teuthology.orchestra.run.smithi040.stdout:(95/131): libgfortran-8.5.0-10.1.el8_6.x86_64.r 2.9 MB/s | 644 kB 00:00 2023-07-14T10:57:23.394 INFO:teuthology.orchestra.run.smithi045.stdout:(81/131): automake-1.16.1-7.el8.noarch.rpm 3.8 MB/s | 713 kB 00:00 2023-07-14T10:57:23.403 INFO:teuthology.orchestra.run.smithi040.stdout:(96/131): NetworkManager-initscripts-updown-1.3 750 kB/s | 138 kB 00:00 2023-07-14T10:57:23.429 INFO:teuthology.orchestra.run.smithi040.stdout:(97/131): libquadmath-8.5.0-10.1.el8_6.x86_64.r 732 kB/s | 171 kB 00:00 2023-07-14T10:57:23.468 INFO:teuthology.orchestra.run.smithi045.stdout:(82/131): python3-babel-2.5.1-7.el8.noarch.rpm 22 MB/s | 4.8 MB 00:00 2023-07-14T10:57:23.484 INFO:teuthology.orchestra.run.smithi040.stdout:(98/131): python3-pyasn1-modules-0.3.7-6.el8.no 1.3 MB/s | 110 kB 00:00 2023-07-14T10:57:23.489 INFO:teuthology.orchestra.run.smithi045.stdout:(83/131): python3-jsonpatch-1.21-2.el8.noarch.r 289 kB/s | 27 kB 00:00 2023-07-14T10:57:23.513 INFO:teuthology.orchestra.run.smithi045.stdout:(84/131): oniguruma-6.8.2-2.el8.x86_64.rpm 574 kB/s | 187 kB 00:00 2023-07-14T10:57:23.529 INFO:teuthology.orchestra.run.smithi040.stdout:(99/131): python3-pytz-2017.2-9.el8.noarch.rpm 541 kB/s | 54 kB 00:00 2023-07-14T10:57:23.609 INFO:teuthology.orchestra.run.smithi040.stdout:(100/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.0 MB/s | 259 kB 00:00 2023-07-14T10:57:23.638 INFO:teuthology.orchestra.run.smithi045.stdout:(85/131): flex-2.6.1-9.el8.x86_64.rpm 1.9 MB/s | 320 kB 00:00 2023-07-14T10:57:23.653 INFO:teuthology.orchestra.run.smithi045.stdout:(86/131): python3-prettytable-0.7.2-14.el8.noar 263 kB/s | 44 kB 00:00 2023-07-14T10:57:23.660 INFO:teuthology.orchestra.run.smithi040.stdout:(101/131): socat-1.7.4.1-1.el8.x86_64.rpm 2.4 MB/s | 323 kB 00:00 2023-07-14T10:57:23.885 INFO:teuthology.orchestra.run.smithi040.stdout:(102/131): python3-scipy-1.0.0-21.module+el8.5. 27 MB/s | 14 MB 00:00 2023-07-14T10:57:23.937 INFO:teuthology.orchestra.run.smithi040.stdout:(103/131): python3-markupsafe-0.23-19.el8.x86_6 118 kB/s | 39 kB 00:00 2023-07-14T10:57:23.977 INFO:teuthology.orchestra.run.smithi040.stdout:(104/131): bison-3.0.4-10.el8.x86_64.rpm 2.1 MB/s | 688 kB 00:00 2023-07-14T10:57:23.978 INFO:teuthology.orchestra.run.smithi045.stdout:(87/131): python3-numpy-1.14.3-10.el8.x86_64.rp 8.0 MB/s | 3.7 MB 00:00 2023-07-14T10:57:24.011 INFO:teuthology.orchestra.run.smithi040.stdout:(105/131): python3-jinja2-2.10.1-3.el8.noarch.r 4.2 MB/s | 538 kB 00:00 2023-07-14T10:57:24.011 INFO:teuthology.orchestra.run.smithi045.stdout:(88/131): python3-numpy-f2py-1.14.3-10.el8.x86_ 628 kB/s | 225 kB 00:00 2023-07-14T10:57:24.044 INFO:teuthology.orchestra.run.smithi040.stdout:(106/131): python3-mako-1.0.6-13.el8.noarch.rpm 1.4 MB/s | 157 kB 00:00 2023-07-14T10:57:24.045 INFO:teuthology.orchestra.run.smithi045.stdout:(89/131): python3-werkzeug-0.12.2-4.el8.noarch. 1.1 MB/s | 457 kB 00:00 2023-07-14T10:57:24.078 INFO:teuthology.orchestra.run.smithi040.stdout:(107/131): oniguruma-6.8.2-2.el8.x86_64.rpm 1.8 MB/s | 187 kB 00:00 2023-07-14T10:57:24.145 INFO:teuthology.orchestra.run.smithi040.stdout:(108/131): automake-1.16.1-7.el8.noarch.rpm 5.2 MB/s | 713 kB 00:00 2023-07-14T10:57:24.165 INFO:teuthology.orchestra.run.smithi045.stdout:(90/131): uuid-1.6.2-43.el8.x86_64.rpm 365 kB/s | 64 kB 00:00 2023-07-14T10:57:24.179 INFO:teuthology.orchestra.run.smithi045.stdout:(91/131): python3-pyasn1-0.3.7-6.el8.noarch.rpm 756 kB/s | 126 kB 00:00 2023-07-14T10:57:24.216 INFO:teuthology.orchestra.run.smithi045.stdout:(92/131): python3-jsonpointer-1.10-11.el8.noarc 120 kB/s | 20 kB 00:00 2023-07-14T10:57:24.245 INFO:teuthology.orchestra.run.smithi040.stdout:(109/131): python3-jsonpatch-1.21-2.el8.noarch. 167 kB/s | 27 kB 00:00 2023-07-14T10:57:24.321 INFO:teuthology.orchestra.run.smithi040.stdout:(110/131): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2023-07-14T10:57:24.348 INFO:teuthology.orchestra.run.smithi040.stdout:(111/131): flex-2.6.1-9.el8.x86_64.rpm 1.6 MB/s | 320 kB 00:00 2023-07-14T10:57:24.371 INFO:teuthology.orchestra.run.smithi040.stdout:(112/131): python3-prettytable-0.7.2-14.el8.noa 329 kB/s | 44 kB 00:00 2023-07-14T10:57:24.421 INFO:teuthology.orchestra.run.smithi045.stdout:(93/131): openblas-threads-0.3.15-3.el8.x86_64. 23 MB/s | 4.9 MB 00:00 2023-07-14T10:57:24.451 INFO:teuthology.orchestra.run.smithi045.stdout:(94/131): python3-pyOpenSSL-19.0.0-1.el8.noarch 354 kB/s | 103 kB 00:00 2023-07-14T10:57:24.472 INFO:teuthology.orchestra.run.smithi040.stdout:(113/131): python3-werkzeug-0.12.2-4.el8.noarch 3.6 MB/s | 457 kB 00:00 2023-07-14T10:57:24.547 INFO:teuthology.orchestra.run.smithi040.stdout:(114/131): python3-numpy-1.14.3-10.el8.x86_64.r 17 MB/s | 3.7 MB 00:00 2023-07-14T10:57:24.559 INFO:teuthology.orchestra.run.smithi045.stdout:(95/131): libtool-2.4.6-25.el8.x86_64.rpm 1.8 MB/s | 709 kB 00:00 2023-07-14T10:57:24.564 INFO:teuthology.orchestra.run.smithi040.stdout:(115/131): uuid-1.6.2-43.el8.x86_64.rpm 697 kB/s | 64 kB 00:00 2023-07-14T10:57:24.580 INFO:teuthology.orchestra.run.smithi045.stdout:(96/131): libpq-13.5-1.el8.x86_64.rpm 1.2 MB/s | 198 kB 00:00 2023-07-14T10:57:24.590 INFO:teuthology.orchestra.run.smithi040.stdout:(116/131): python3-numpy-f2py-1.14.3-10.el8.x86 1.0 MB/s | 225 kB 00:00 2023-07-14T10:57:24.657 INFO:teuthology.orchestra.run.smithi040.stdout:(117/131): python3-pyasn1-0.3.7-6.el8.noarch.rp 1.1 MB/s | 126 kB 00:00 2023-07-14T10:57:24.680 INFO:teuthology.orchestra.run.smithi045.stdout:(97/131): openblas-0.3.15-3.el8.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2023-07-14T10:57:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:(118/131): python3-jsonpointer-1.10-11.el8.noar 171 kB/s | 20 kB 00:00 2023-07-14T10:57:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:(119/131): python3-pyOpenSSL-19.0.0-1.el8.noarc 824 kB/s | 103 kB 00:00 2023-07-14T10:57:24.740 INFO:teuthology.orchestra.run.smithi045.stdout:(98/131): librdkafka-0.11.4-3.el8.x86_64.rpm 2.3 MB/s | 354 kB 00:00 2023-07-14T10:57:24.756 INFO:teuthology.orchestra.run.smithi045.stdout:(99/131): jq-1.6-3.el8.x86_64.rpm 1.0 MB/s | 202 kB 00:00 2023-07-14T10:57:24.807 INFO:teuthology.orchestra.run.smithi040.stdout:(120/131): libtool-2.4.6-25.el8.x86_64.rpm 4.6 MB/s | 709 kB 00:00 2023-07-14T10:57:24.929 INFO:teuthology.orchestra.run.smithi045.stdout:(100/131): postgresql-server-10.21-2.module+el8 21 MB/s | 5.1 MB 00:00 2023-07-14T10:57:24.956 INFO:teuthology.orchestra.run.smithi045.stdout:(101/131): postgresql-contrib-10.21-2.module+el 3.5 MB/s | 808 kB 00:00 2023-07-14T10:57:24.981 INFO:teuthology.orchestra.run.smithi040.stdout:(121/131): openblas-threads-0.3.15-3.el8.x86_64 17 MB/s | 4.9 MB 00:00 2023-07-14T10:57:25.008 INFO:teuthology.orchestra.run.smithi040.stdout:(122/131): libpq-13.5-1.el8.x86_64.rpm 676 kB/s | 198 kB 00:00 2023-07-14T10:57:25.066 INFO:teuthology.orchestra.run.smithi045.stdout:(102/131): postgresql-10.21-2.module+el8.6.0+15 4.9 MB/s | 1.5 MB 00:00 2023-07-14T10:57:25.141 INFO:teuthology.orchestra.run.smithi040.stdout:(123/131): openblas-0.3.15-3.el8.x86_64.rpm 14 MB/s | 4.7 MB 00:00 2023-07-14T10:57:25.157 INFO:teuthology.orchestra.run.smithi045.stdout:(103/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 200 kB/s | 47 kB 00:00 2023-07-14T10:57:25.175 INFO:teuthology.orchestra.run.smithi040.stdout:(124/131): jq-1.6-3.el8.x86_64.rpm 1.0 MB/s | 202 kB 00:00 2023-07-14T10:57:25.183 INFO:teuthology.orchestra.run.smithi045.stdout:(104/131): libselinux-devel-2.9-5.el8.x86_64.rp 885 kB/s | 200 kB 00:00 2023-07-14T10:57:25.201 INFO:teuthology.orchestra.run.smithi040.stdout:(125/131): librdkafka-0.11.4-3.el8.x86_64.rpm 1.8 MB/s | 354 kB 00:00 2023-07-14T10:57:25.211 INFO:teuthology.orchestra.run.smithi045.stdout:(105/131): python3-jwt-1.6.1-2.el8.noarch.rpm 300 kB/s | 43 kB 00:00 2023-07-14T10:57:25.285 INFO:teuthology.orchestra.run.smithi040.stdout:(126/131): postgresql-contrib-10.21-2.module+el 7.3 MB/s | 808 kB 00:00 2023-07-14T10:57:25.333 INFO:teuthology.orchestra.run.smithi045.stdout:(106/131): keyutils-libs-devel-1.5.10-9.el8.x86 320 kB/s | 48 kB 00:00 2023-07-14T10:57:25.360 INFO:teuthology.orchestra.run.smithi045.stdout:(107/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 1.1 MB/s | 220 kB 00:00 2023-07-14T10:57:25.425 INFO:teuthology.orchestra.run.smithi040.stdout:(127/131): postgresql-server-10.21-2.module+el8 18 MB/s | 5.1 MB 00:00 2023-07-14T10:57:25.459 INFO:teuthology.orchestra.run.smithi045.stdout:(108/131): python3-requests-oauthlib-1.0.0-1.el 341 kB/s | 43 kB 00:00 2023-07-14T10:57:25.483 INFO:teuthology.orchestra.run.smithi040.stdout:(128/131): postgresql-10.21-2.module+el8.6.0+15 5.3 MB/s | 1.5 MB 00:00 2023-07-14T10:57:25.484 INFO:teuthology.orchestra.run.smithi045.stdout:(109/131): libsepol-devel-2.9-3.el8.x86_64.rpm 315 kB/s | 87 kB 00:00 2023-07-14T10:57:25.511 INFO:teuthology.orchestra.run.smithi045.stdout:(110/131): python3-pycparser-2.14-14.el8.noarch 725 kB/s | 109 kB 00:00 2023-07-14T10:57:25.620 INFO:teuthology.orchestra.run.smithi045.stdout:(111/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 1.4 MB/s | 229 kB 00:00 2023-07-14T10:57:25.660 INFO:teuthology.orchestra.run.smithi045.stdout:(112/131): libaio-devel-0.3.112-1.el8.x86_64.rp 124 kB/s | 19 kB 00:00 2023-07-14T10:57:25.711 INFO:teuthology.orchestra.run.smithi045.stdout:(113/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 2.4 MB/s | 560 kB 00:00 2023-07-14T10:57:25.771 INFO:teuthology.orchestra.run.smithi045.stdout:(114/131): python3-oauthlib-2.1.0-1.el8.noarch. 1.0 MB/s | 155 kB 00:00 2023-07-14T10:57:25.802 INFO:teuthology.orchestra.run.smithi045.stdout:(115/131): python3-cryptography-3.2.1-5.el8.x86 3.8 MB/s | 559 kB 00:00 2023-07-14T10:57:25.834 INFO:teuthology.orchestra.run.smithi040.stdout:(129/131): librados2-18.0.0-4757.g3fc0089c.el8. 6.7 MB/s | 3.7 MB 00:00 2023-07-14T10:57:25.840 INFO:teuthology.orchestra.run.smithi045.stdout:(116/131): libverto-devel-0.3.0-5.el8.x86_64.rp 143 kB/s | 18 kB 00:00 2023-07-14T10:57:25.903 INFO:teuthology.orchestra.run.smithi045.stdout:(117/131): pcre2-devel-10.32-2.el8.x86_64.rpm 4.4 MB/s | 605 kB 00:00 2023-07-14T10:57:25.903 INFO:teuthology.orchestra.run.smithi040.stdout:(130/131): librbd1-18.0.0-4757.g3fc0089c.el8.x8 7.5 MB/s | 3.6 MB 00:00 2023-07-14T10:57:25.929 INFO:teuthology.orchestra.run.smithi045.stdout:(118/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.8 MB/s | 238 kB 00:00 2023-07-14T10:57:25.937 INFO:teuthology.orchestra.run.smithi040.stdout:(131/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 1.7 MB/s | 798 kB 00:00 2023-07-14T10:57:25.938 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:57:25.939 INFO:teuthology.orchestra.run.smithi040.stdout:Total 13 MB/s | 205 MB 00:15 2023-07-14T10:57:26.028 INFO:teuthology.orchestra.run.smithi045.stdout:(119/131): openssl-devel-1.1.1k-7.el8_6.x86_64. 12 MB/s | 2.3 MB 00:00 2023-07-14T10:57:26.054 INFO:teuthology.orchestra.run.smithi045.stdout:(120/131): xfsprogs-devel-5.0.0-10.el8.x86_64.r 480 kB/s | 72 kB 00:00 2023-07-14T10:57:26.080 INFO:teuthology.orchestra.run.smithi045.stdout:(121/131): fuse-2.9.7-15.el8.x86_64.rpm 550 kB/s | 83 kB 00:00 2023-07-14T10:57:26.141 INFO:teuthology.orchestra.run.smithi045.stdout:(122/131): zlib-devel-1.2.11-18.el8_5.x86_64.rp 988 kB/s | 58 kB 00:00 2023-07-14T10:57:26.156 INFO:teuthology.orchestra.run.smithi045.stdout:(123/131): elfutils-libelf-devel-0.186-1.el8.x8 476 kB/s | 60 kB 00:00 2023-07-14T10:57:26.224 INFO:teuthology.orchestra.run.smithi045.stdout:(124/131): libuuid-devel-2.32.1-35.el8.x86_64.r 586 kB/s | 98 kB 00:00 2023-07-14T10:57:26.322 INFO:teuthology.orchestra.run.smithi045.stdout:(125/131): libquadmath-8.5.0-10.1.el8_6.x86_64. 1.7 MB/s | 171 kB 00:00 2023-07-14T10:57:26.339 INFO:teuthology.orchestra.run.smithi045.stdout:(126/131): libcom_err-devel-1.45.6-4.el8.x86_64 192 kB/s | 39 kB 00:00 2023-07-14T10:57:26.382 INFO:teuthology.orchestra.run.smithi045.stdout:(127/131): libgfortran-8.5.0-10.1.el8_6.x86_64. 2.8 MB/s | 644 kB 00:00 2023-07-14T10:57:26.442 INFO:teuthology.orchestra.run.smithi045.stdout:(128/131): NetworkManager-initscripts-updown-1. 1.1 MB/s | 138 kB 00:00 2023-07-14T10:57:26.599 INFO:teuthology.orchestra.run.smithi045.stdout:(129/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 5.0 MB/s | 798 kB 00:00 2023-07-14T10:57:26.877 INFO:teuthology.orchestra.run.smithi045.stdout:(130/131): librados2-18.0.0-4757.g3fc0089c.el8. 6.9 MB/s | 3.7 MB 00:00 2023-07-14T10:57:27.588 INFO:teuthology.orchestra.run.smithi045.stdout:(131/131): librbd1-18.0.0-4757.g3fc0089c.el8.x8 2.9 MB/s | 3.6 MB 00:01 2023-07-14T10:57:27.590 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:57:27.591 INFO:teuthology.orchestra.run.smithi045.stdout:Total 9.9 MB/s | 205 MB 00:20 2023-07-14T10:57:27.623 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-07-14T10:57:27.769 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-07-14T10:57:27.770 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-07-14T10:57:28.453 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2023-07-14T10:57:28.583 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2023-07-14T10:57:28.586 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2023-07-14T10:57:29.353 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-07-14T10:57:29.356 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-07-14T10:57:29.763 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2023-07-14T10:57:29.770 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2023-07-14T10:57:34.442 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-07-14T10:57:34.630 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-07-14T10:57:34.745 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2023-07-14T10:57:34.848 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-07-14T10:57:34.913 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-07-14T10:57:35.037 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 3/134 2023-07-14T10:57:35.112 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-07-14T10:57:35.199 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 3/134 2023-07-14T10:57:35.316 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 3/134 2023-07-14T10:57:35.395 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-07-14T10:57:35.473 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 4/134 2023-07-14T10:57:35.589 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 4/134 2023-07-14T10:57:35.840 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 5/134 2023-07-14T10:57:35.917 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-07-14T10:57:36.154 INFO:teuthology.orchestra.run.smithi040.stdout: Upgrading : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 5/134 2023-07-14T10:57:36.279 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 5/134 2023-07-14T10:57:36.456 INFO:teuthology.orchestra.run.smithi045.stdout: Upgrading : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:57:36.641 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 5/134 2023-07-14T10:57:36.822 INFO:teuthology.orchestra.run.smithi040.stdout: Upgrading : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:57:36.883 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:57:36.979 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:57:37.092 INFO:teuthology.orchestra.run.smithi045.stdout: Upgrading : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:57:37.212 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:57:37.240 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:57:37.356 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:57:37.416 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:57:37.523 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:57:37.526 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:57:37.698 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 9/134 2023-07-14T10:57:37.766 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 9/134 2023-07-14T10:57:37.925 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-07-14T10:57:38.168 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-07-14T10:57:38.282 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-07-14T10:57:38.537 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-07-14T10:57:38.570 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-07-14T10:57:38.800 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-07-14T10:57:38.931 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:57:38.969 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-07-14T10:57:39.040 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:57:39.193 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-07-14T10:57:39.197 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:57:39.289 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:57:39.345 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:57:39.408 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 16/134 2023-07-14T10:57:39.489 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:57:39.548 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 16/134 2023-07-14T10:57:39.646 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:57:39.738 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 17/134 2023-07-14T10:57:39.824 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:57:40.065 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 16/134 2023-07-14T10:57:40.247 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 17/134 2023-07-14T10:57:40.381 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 17/134 2023-07-14T10:57:40.437 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 18/134 2023-07-14T10:57:40.562 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-07-14T10:57:40.563 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 18/134 2023-07-14T10:57:40.775 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-07-14T10:57:40.915 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-07-14T10:57:41.199 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 18/134 2023-07-14T10:57:41.420 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : openblas-0.3.15-3.el8.x86_64 19/134 2023-07-14T10:57:41.653 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-07-14T10:57:41.861 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-07-14T10:57:42.183 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 19/134 2023-07-14T10:57:42.457 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 20/134 2023-07-14T10:57:42.593 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-07-14T10:57:42.698 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 21/134 2023-07-14T10:57:42.752 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : openblas-0.3.15-3.el8.x86_64 22/134 2023-07-14T10:57:42.909 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 22/134 2023-07-14T10:57:43.089 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 23/134 2023-07-14T10:57:43.345 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 24/134 2023-07-14T10:57:43.575 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-07-14T10:57:43.661 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 22/134 2023-07-14T10:57:43.771 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 26/134 2023-07-14T10:57:43.889 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 27/134 2023-07-14T10:57:43.905 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 23/134 2023-07-14T10:57:44.118 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 24/134 2023-07-14T10:57:44.394 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-07-14T10:57:44.560 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 26/134 2023-07-14T10:57:44.702 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 27/134 2023-07-14T10:57:44.830 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 28/134 2023-07-14T10:57:44.978 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 28/134 2023-07-14T10:57:45.161 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-07-14T10:57:45.162 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 29/134 2023-07-14T10:57:45.438 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 30/134 2023-07-14T10:57:45.687 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 31/134 2023-07-14T10:57:45.863 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 32/134 2023-07-14T10:57:47.360 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-07-14T10:57:47.759 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 32/134 2023-07-14T10:57:48.060 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 33/134 2023-07-14T10:57:48.253 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-07-14T10:57:48.450 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-07-14T10:57:48.584 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 30/134 2023-07-14T10:57:48.630 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 36/134 2023-07-14T10:57:48.783 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 37/134 2023-07-14T10:57:48.968 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 31/134 2023-07-14T10:57:48.988 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 38/134 2023-07-14T10:57:49.156 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 39/134 2023-07-14T10:57:49.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 32/134 2023-07-14T10:57:49.331 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : fuse-2.9.7-15.el8.x86_64 40/134 2023-07-14T10:57:49.507 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 33/134 2023-07-14T10:57:49.508 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 41/134 2023-07-14T10:57:49.737 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-07-14T10:57:49.749 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 42/134 2023-07-14T10:57:49.940 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 43/134 2023-07-14T10:57:49.950 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-07-14T10:57:50.147 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 36/134 2023-07-14T10:57:50.360 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 37/134 2023-07-14T10:57:50.376 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 44/134 2023-07-14T10:57:50.554 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 38/134 2023-07-14T10:57:50.608 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 45/134 2023-07-14T10:57:50.725 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 39/134 2023-07-14T10:57:50.817 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 46/134 2023-07-14T10:57:50.904 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : uuid-1.6.2-43.el8.x86_64 40/134 2023-07-14T10:57:50.996 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 40/134 2023-07-14T10:57:51.028 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 47/134 2023-07-14T10:57:51.284 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 48/134 2023-07-14T10:57:51.293 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 41/134 2023-07-14T10:57:51.488 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : automake-1.16.1-7.el8.noarch 42/134 2023-07-14T10:57:51.606 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 49/134 2023-07-14T10:57:51.669 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 43/134 2023-07-14T10:57:51.773 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 43/134 2023-07-14T10:57:51.847 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 50/134 2023-07-14T10:57:52.007 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : jq-1.6-3.el8.x86_64 44/134 2023-07-14T10:57:52.053 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 51/134 2023-07-14T10:57:52.187 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 45/134 2023-07-14T10:57:52.269 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 52/134 2023-07-14T10:57:52.368 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 46/134 2023-07-14T10:57:52.478 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 53/134 2023-07-14T10:57:52.541 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 47/134 2023-07-14T10:57:52.636 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 54/134 2023-07-14T10:57:52.732 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : fuse-2.9.7-15.el8.x86_64 48/134 2023-07-14T10:57:52.918 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 49/134 2023-07-14T10:57:52.969 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 55/134 2023-07-14T10:57:53.137 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 50/134 2023-07-14T10:57:53.220 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 56/134 2023-07-14T10:57:53.358 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 51/134 2023-07-14T10:57:53.457 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 57/134 2023-07-14T10:57:53.695 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 58/134 2023-07-14T10:57:53.775 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 52/134 2023-07-14T10:57:53.866 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 59/134 2023-07-14T10:57:54.013 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 60/134 2023-07-14T10:57:54.127 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 53/134 2023-07-14T10:57:54.227 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : uuid-1.6.2-43.el8.x86_64 61/134 2023-07-14T10:57:54.308 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 61/134 2023-07-14T10:57:54.335 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 54/134 2023-07-14T10:57:54.547 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 62/134 2023-07-14T10:57:54.626 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 55/134 2023-07-14T10:57:54.760 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : automake-1.16.1-7.el8.noarch 63/134 2023-07-14T10:57:54.886 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 64/134 2023-07-14T10:57:54.977 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 64/134 2023-07-14T10:57:54.990 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 56/134 2023-07-14T10:57:55.177 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : jq-1.6-3.el8.x86_64 65/134 2023-07-14T10:57:55.368 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 57/134 2023-07-14T10:57:55.395 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 66/134 2023-07-14T10:57:55.569 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 58/134 2023-07-14T10:57:55.570 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-07-14T10:57:55.760 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-07-14T10:57:55.785 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 59/134 2023-07-14T10:57:56.007 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 60/134 2023-07-14T10:57:56.050 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-07-14T10:57:56.200 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 61/134 2023-07-14T10:57:56.260 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 70/134 2023-07-14T10:57:56.384 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 62/134 2023-07-14T10:57:56.441 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-07-14T10:57:56.586 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-07-14T10:57:56.729 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 63/134 2023-07-14T10:57:56.766 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-07-14T10:57:56.992 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-07-14T10:57:57.038 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 64/134 2023-07-14T10:57:57.161 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-07-14T10:57:57.314 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-07-14T10:57:57.348 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 65/134 2023-07-14T10:57:57.444 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-07-14T10:57:57.624 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 66/134 2023-07-14T10:57:57.690 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-07-14T10:57:57.812 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-07-14T10:57:57.924 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-07-14T10:57:57.996 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-07-14T10:57:58.135 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-07-14T10:57:58.352 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-07-14T10:57:58.365 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-07-14T10:57:58.543 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 70/134 2023-07-14T10:57:58.563 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-07-14T10:57:58.742 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-07-14T10:57:58.753 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-07-14T10:57:58.990 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-07-14T10:57:59.025 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-07-14T10:57:59.230 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-07-14T10:57:59.471 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-07-14T10:57:59.681 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-07-14T10:57:59.909 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-07-14T10:58:00.074 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-07-14T10:58:00.136 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-07-14T10:58:00.311 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-07-14T10:58:00.418 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-07-14T10:58:00.576 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-07-14T10:58:00.716 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-07-14T10:58:00.768 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-07-14T10:58:01.024 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-07-14T10:58:01.052 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-07-14T10:58:01.279 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-07-14T10:58:01.336 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-07-14T10:58:01.464 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-07-14T10:58:01.530 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-07-14T10:58:01.743 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-07-14T10:58:02.385 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-07-14T10:58:02.544 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T10:58:02.616 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T10:58:02.808 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 93/134 2023-07-14T10:58:03.168 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-07-14T10:58:03.515 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-07-14T10:58:03.761 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-07-14T10:58:04.145 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-07-14T10:58:04.418 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-07-14T10:58:04.672 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-07-14T10:58:05.698 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-07-14T10:58:06.001 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T10:58:06.077 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T10:58:06.293 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 93/134 2023-07-14T10:58:06.684 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:06.934 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:07.562 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:07.816 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T10:58:07.900 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T10:58:08.041 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T10:58:11.549 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T10:58:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T10:58:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T10:58:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T10:58:27.493 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T10:58:27.493 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /sys 2023-07-14T10:58:27.493 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /proc 2023-07-14T10:58:27.493 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /dev 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /run 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /mnt 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /var/tmp 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /home 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /tmp 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout:skipping the directory /dev 2023-07-14T10:58:27.494 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:58:27.860 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T10:58:29.525 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T10:58:29.697 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T10:58:29.931 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T10:58:30.131 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T10:58:30.616 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T10:58:30.748 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T10:58:30.765 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T10:58:32.528 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T10:58:32.706 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T10:58:32.829 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 102/134 2023-07-14T10:58:33.064 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 103/134 2023-07-14T10:58:33.247 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T10:58:33.471 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T10:58:33.775 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T10:58:34.483 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T10:58:34.483 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /sys 2023-07-14T10:58:34.483 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /proc 2023-07-14T10:58:34.483 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /dev 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /run 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /mnt 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /var/tmp 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /home 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /tmp 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /dev 2023-07-14T10:58:34.484 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:58:34.791 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T10:58:34.868 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T10:58:34.950 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T10:58:35.043 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T10:58:35.430 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 107/134 2023-07-14T10:58:35.624 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T10:58:35.659 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T10:58:35.851 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T10:58:35.859 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T10:58:36.708 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T10:58:36.885 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T10:58:37.166 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T10:58:37.232 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 110/134 2023-07-14T10:58:37.374 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T10:58:37.407 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T10:58:37.442 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T10:58:37.612 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T10:58:37.658 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T10:58:37.832 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T10:58:37.932 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T10:58:38.130 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T10:58:38.195 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T10:58:38.195 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T10:58:38.349 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T10:58:41.130 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T10:58:41.366 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T10:58:41.493 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 102/134 2023-07-14T10:58:41.739 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 103/134 2023-07-14T10:58:41.953 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T10:58:42.243 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T10:58:42.456 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T10:58:43.213 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T10:58:43.516 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 115/134 2023-07-14T10:58:43.644 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T10:58:43.662 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libtool-2.4.6-25.el8.x86_64 116/134 2023-07-14T10:58:43.812 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T10:58:43.867 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 116/134 2023-07-14T10:58:43.946 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T10:58:44.365 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 107/134 2023-07-14T10:58:44.588 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T10:58:44.651 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T10:58:44.785 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 117/134 2023-07-14T10:58:44.979 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T10:58:45.007 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T10:58:45.152 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 118/134 2023-07-14T10:58:45.315 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 119/134 2023-07-14T10:58:45.516 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 120/134 2023-07-14T10:58:45.700 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 121/134 2023-07-14T10:58:45.831 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-07-14T10:58:46.491 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 110/134 2023-07-14T10:58:46.726 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T10:58:46.797 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T10:58:46.953 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T10:58:47.032 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T10:58:47.257 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T10:58:47.691 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T10:58:47.865 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T10:58:48.138 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:58:48.369 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:58:48.390 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:58:48.579 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 124/134 2023-07-14T10:58:48.857 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 125/134 2023-07-14T10:58:49.189 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 126/134 2023-07-14T10:58:49.446 INFO:teuthology.orchestra.run.smithi045.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-07-14T10:58:49.586 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-07-14T10:58:49.595 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-07-14T10:58:49.818 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 129/134 2023-07-14T10:58:49.994 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : flex-2.6.1-9.el8.x86_64 130/134 2023-07-14T10:58:50.084 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 130/134 2023-07-14T10:58:50.336 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : bison-3.0.4-10.el8.x86_64 131/134 2023-07-14T10:58:50.350 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 131/134 2023-07-14T10:58:50.350 INFO:teuthology.orchestra.run.smithi045.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:58:50.540 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:58:50.541 INFO:teuthology.orchestra.run.smithi045.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T10:58:50.821 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T10:58:50.994 INFO:teuthology.orchestra.run.smithi045.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:58:51.516 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:58:51.516 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 1/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 3/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 4/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 5/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:58:51.517 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 9/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 10/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 11/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 12/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 13/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 14/134 2023-07-14T10:58:51.518 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 15/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 16/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 17/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 18/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 19/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 20/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 21/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 22/134 2023-07-14T10:58:51.519 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 23/134 2023-07-14T10:58:51.520 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 24/134 2023-07-14T10:58:51.520 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 25/134 2023-07-14T10:58:51.520 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 26/134 2023-07-14T10:58:51.521 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 27/134 2023-07-14T10:58:51.521 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 28/134 2023-07-14T10:58:51.521 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 29/134 2023-07-14T10:58:51.540 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 30/134 2023-07-14T10:58:51.541 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 31/134 2023-07-14T10:58:51.541 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 32/134 2023-07-14T10:58:51.541 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 33/134 2023-07-14T10:58:51.541 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 34/134 2023-07-14T10:58:51.541 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 35/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 36/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 37/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 38/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-07-14T10:58:51.542 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-07-14T10:58:51.543 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-07-14T10:58:51.543 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-07-14T10:58:51.544 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-07-14T10:58:51.544 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-07-14T10:58:51.544 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-07-14T10:58:51.544 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-07-14T10:58:51.545 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-07-14T10:58:51.547 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-07-14T10:58:51.547 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-07-14T10:58:51.547 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-07-14T10:58:51.548 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-07-14T10:58:51.550 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 72/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 73/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 74/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 75/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 76/134 2023-07-14T10:58:51.551 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 77/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : bison-3.0.4-10.el8.x86_64 78/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 79/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 80/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 81/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : automake-1.16.1-7.el8.noarch 82/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 83/134 2023-07-14T10:58:51.552 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 84/134 2023-07-14T10:58:51.553 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : flex-2.6.1-9.el8.x86_64 85/134 2023-07-14T10:58:51.553 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 86/134 2023-07-14T10:58:51.553 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 87/134 2023-07-14T10:58:51.553 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 88/134 2023-07-14T10:58:51.553 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 89/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 90/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 91/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 92/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 93/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 94/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 95/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libpq-13.5-1.el8.x86_64 96/134 2023-07-14T10:58:51.554 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 97/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : jq-1.6-3.el8.x86_64 98/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 99/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 100/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 101/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 102/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 103/134 2023-07-14T10:58:51.555 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 104/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 105/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 106/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 107/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 108/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 109/134 2023-07-14T10:58:51.556 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 110/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 111/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 112/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 113/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 114/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 115/134 2023-07-14T10:58:51.557 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 116/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 117/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 118/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 119/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 120/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 121/134 2023-07-14T10:58:51.558 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-07-14T10:58:51.559 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 123/134 2023-07-14T10:58:51.559 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 124/134 2023-07-14T10:58:51.559 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 125/134 2023-07-14T10:58:51.559 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 126/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 127/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 128/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 129/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 131/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:58:51.560 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-07-14T10:58:53.238 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T10:58:54.010 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:58:54.010 INFO:teuthology.orchestra.run.smithi045.stdout:Last metadata expiration check: 0:02:00 ago on Fri 14 Jul 2023 10:56:54 AM UTC. 2023-07-14T10:58:54.706 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 115/134 2023-07-14T10:58:55.069 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 116/134 2023-07-14T10:58:55.268 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 117/134 2023-07-14T10:58:55.573 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 118/134 2023-07-14T10:58:55.735 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libtool-2.4.6-25.el8.x86_64 119/134 2023-07-14T10:58:56.012 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 119/134 2023-07-14T10:58:56.240 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 120/134 2023-07-14T10:58:56.452 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 121/134 2023-07-14T10:58:56.622 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-07-14T10:59:00.113 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:59:00.364 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:59:00.405 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T10:59:00.644 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 124/134 2023-07-14T10:59:00.836 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 125/134 2023-07-14T10:59:01.173 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 126/134 2023-07-14T10:59:01.516 INFO:teuthology.orchestra.run.smithi040.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-07-14T10:59:01.735 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : flex-2.6.1-9.el8.x86_64 128/134 2023-07-14T10:59:01.849 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 128/134 2023-07-14T10:59:02.061 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : bison-3.0.4-10.el8.x86_64 129/134 2023-07-14T10:59:02.088 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 129/134 2023-07-14T10:59:02.292 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-07-14T10:59:02.316 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-07-14T10:59:02.470 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 131/134 2023-07-14T10:59:02.471 INFO:teuthology.orchestra.run.smithi040.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:59:02.849 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:59:02.850 INFO:teuthology.orchestra.run.smithi040.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T10:59:03.116 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T10:59:03.321 INFO:teuthology.orchestra.run.smithi040.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:59:03.594 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:59:03.595 INFO:teuthology.orchestra.run.smithi040.stdout:Couldn't write '0' to 'kernel/yama/ptrace_scope', ignoring: No such file or directory 2023-07-14T10:59:03.595 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:59:04.045 INFO:teuthology.orchestra.run.smithi045.stdout:Installed products updated. 2023-07-14T10:59:04.240 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 1/134 2023-07-14T10:59:04.240 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2/134 2023-07-14T10:59:04.241 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 3/134 2023-07-14T10:59:04.241 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 4/134 2023-07-14T10:59:04.244 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 5/134 2023-07-14T10:59:04.244 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:59:04.244 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:59:04.244 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:59:04.244 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 9/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 10/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 11/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 12/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 13/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 14/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 15/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 16/134 2023-07-14T10:59:04.245 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 17/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 18/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 19/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 20/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 21/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 22/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 23/134 2023-07-14T10:59:04.246 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 24/134 2023-07-14T10:59:04.247 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 25/134 2023-07-14T10:59:04.249 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 26/134 2023-07-14T10:59:04.249 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 27/134 2023-07-14T10:59:04.250 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 28/134 2023-07-14T10:59:04.250 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 29/134 2023-07-14T10:59:04.250 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 30/134 2023-07-14T10:59:04.263 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 31/134 2023-07-14T10:59:04.263 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 32/134 2023-07-14T10:59:04.263 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 33/134 2023-07-14T10:59:04.263 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 34/134 2023-07-14T10:59:04.263 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 35/134 2023-07-14T10:59:04.264 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 36/134 2023-07-14T10:59:04.264 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 37/134 2023-07-14T10:59:04.264 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 38/134 2023-07-14T10:59:04.264 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-07-14T10:59:04.265 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-07-14T10:59:04.265 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-07-14T10:59:04.265 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-07-14T10:59:04.265 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-07-14T10:59:04.266 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-07-14T10:59:04.267 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-07-14T10:59:04.268 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-07-14T10:59:04.269 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 72/134 2023-07-14T10:59:04.270 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 73/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 74/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 75/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 76/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 77/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 78/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 79/134 2023-07-14T10:59:04.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 80/134 2023-07-14T10:59:04.272 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 81/134 2023-07-14T10:59:04.272 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 82/134 2023-07-14T10:59:04.272 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 83/134 2023-07-14T10:59:04.272 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 84/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 85/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 86/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 87/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 88/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 89/134 2023-07-14T10:59:04.273 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 90/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 91/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 92/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 93/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 94/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 95/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 96/134 2023-07-14T10:59:04.274 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 97/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 98/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 99/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 100/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 101/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 102/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 103/134 2023-07-14T10:59:04.275 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : bison-3.0.4-10.el8.x86_64 104/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 105/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 106/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 107/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : automake-1.16.1-7.el8.noarch 108/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 109/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 110/134 2023-07-14T10:59:04.276 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : flex-2.6.1-9.el8.x86_64 111/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 112/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 113/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 114/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 115/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 116/134 2023-07-14T10:59:04.277 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 117/134 2023-07-14T10:59:04.278 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 118/134 2023-07-14T10:59:04.278 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 119/134 2023-07-14T10:59:04.278 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 120/134 2023-07-14T10:59:04.278 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 121/134 2023-07-14T10:59:04.278 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libpq-13.5-1.el8.x86_64 122/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 123/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : jq-1.6-3.el8.x86_64 124/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 125/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 126/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 127/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 128/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 129/134 2023-07-14T10:59:04.279 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-07-14T10:59:04.280 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 131/134 2023-07-14T10:59:04.280 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T10:59:04.280 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-07-14T10:59:04.281 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:59:04.281 INFO:teuthology.orchestra.run.smithi045.stdout:Upgraded: 2023-07-14T10:59:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-07-14T10:59:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.282 INFO:teuthology.orchestra.run.smithi045.stdout: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.282 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: automake-1.16.1-7.el8.noarch 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: bison-3.0.4-10.el8.x86_64 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.283 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-immutable-object-cache-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.284 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.285 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.286 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: flex-2.6.1-9.el8.x86_64 2023-07-14T10:59:04.287 INFO:teuthology.orchestra.run.smithi045.stdout: fuse-2.9.7-15.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: jq-1.6-3.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-07-14T10:59:04.288 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: liboath-2.6.2-3.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libpq-13.5-1.el8.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-07-14T10:59:04.289 INFO:teuthology.orchestra.run.smithi045.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-07-14T10:59:04.291 INFO:teuthology.orchestra.run.smithi045.stdout: librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-07-14T10:59:04.292 INFO:teuthology.orchestra.run.smithi045.stdout: libtool-2.4.6-25.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: libunwind-1.3.1-3.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-0.3.15-3.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-07-14T10:59:04.293 INFO:teuthology.orchestra.run.smithi045.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:04.294 INFO:teuthology.orchestra.run.smithi045.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-babel-2.5.1-7.el8.noarch 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.295 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-07-14T10:59:04.296 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-6.2-6.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-07-14T10:59:04.297 INFO:teuthology.orchestra.run.smithi045.stdout: python3-mako-1.0.6-13.el8.noarch 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-07-14T10:59:04.298 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-07-14T10:59:04.300 INFO:teuthology.orchestra.run.smithi045.stdout: python3-portend-2.6-1.el8.noarch 2023-07-14T10:59:04.300 INFO:teuthology.orchestra.run.smithi045.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pycparser-2.14-14.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-pytz-2017.2-9.el8.noarch 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.301 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-routes-2.4.1-12.el8.noarch 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-rsa-4.8-1.el8.noarch 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-07-14T10:59:04.302 INFO:teuthology.orchestra.run.smithi045.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-07-14T10:59:04.303 INFO:teuthology.orchestra.run.smithi045.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: socat-1.7.4.1-1.el8.x86_64 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: thrift-0.13.0-2.el8.x86_64 2023-07-14T10:59:04.304 INFO:teuthology.orchestra.run.smithi045.stdout: uuid-1.6.2-43.el8.x86_64 2023-07-14T10:59:04.305 INFO:teuthology.orchestra.run.smithi045.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-07-14T10:59:04.305 INFO:teuthology.orchestra.run.smithi045.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-07-14T10:59:04.305 INFO:teuthology.orchestra.run.smithi045.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-07-14T10:59:04.305 INFO:teuthology.orchestra.run.smithi045.stdout: 2023-07-14T10:59:04.305 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2023-07-14T10:59:04.455 DEBUG:teuthology.parallel:result is None 2023-07-14T10:59:07.018 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T10:59:07.019 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:11 ago on Fri 14 Jul 2023 10:56:56 AM UTC. 2023-07-14T10:59:16.837 INFO:teuthology.orchestra.run.smithi040.stdout:Installed products updated. 2023-07-14T10:59:17.077 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:59:17.078 INFO:teuthology.orchestra.run.smithi040.stdout:Upgraded: 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-07-14T10:59:17.082 INFO:teuthology.orchestra.run.smithi040.stdout: automake-1.16.1-7.el8.noarch 2023-07-14T10:59:17.083 INFO:teuthology.orchestra.run.smithi040.stdout: bison-3.0.4-10.el8.x86_64 2023-07-14T10:59:17.083 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.083 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.083 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.084 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.084 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.084 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-immutable-object-cache-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.085 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.085 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.085 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.085 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.085 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.086 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: flex-2.6.1-9.el8.x86_64 2023-07-14T10:59:17.087 INFO:teuthology.orchestra.run.smithi040.stdout: fuse-2.9.7-15.el8.x86_64 2023-07-14T10:59:17.088 INFO:teuthology.orchestra.run.smithi040.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-07-14T10:59:17.088 INFO:teuthology.orchestra.run.smithi040.stdout: jq-1.6-3.el8.x86_64 2023-07-14T10:59:17.088 INFO:teuthology.orchestra.run.smithi040.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-07-14T10:59:17.089 INFO:teuthology.orchestra.run.smithi040.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: liboath-2.6.2-3.el8.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: libpq-13.5-1.el8.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.090 INFO:teuthology.orchestra.run.smithi040.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-07-14T10:59:17.091 INFO:teuthology.orchestra.run.smithi040.stdout: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.091 INFO:teuthology.orchestra.run.smithi040.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-07-14T10:59:17.091 INFO:teuthology.orchestra.run.smithi040.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-07-14T10:59:17.091 INFO:teuthology.orchestra.run.smithi040.stdout: libtool-2.4.6-25.el8.x86_64 2023-07-14T10:59:17.092 INFO:teuthology.orchestra.run.smithi040.stdout: libunwind-1.3.1-3.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-0.3.15-3.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-07-14T10:59:17.098 INFO:teuthology.orchestra.run.smithi040.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T10:59:17.099 INFO:teuthology.orchestra.run.smithi040.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-07-14T10:59:17.100 INFO:teuthology.orchestra.run.smithi040.stdout: python3-babel-2.5.1-7.el8.noarch 2023-07-14T10:59:17.100 INFO:teuthology.orchestra.run.smithi040.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-07-14T10:59:17.100 INFO:teuthology.orchestra.run.smithi040.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-07-14T10:59:17.100 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-07-14T10:59:17.101 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-07-14T10:59:17.102 INFO:teuthology.orchestra.run.smithi040.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-07-14T10:59:17.102 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-6.2-6.el8.noarch 2023-07-14T10:59:17.102 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-07-14T10:59:17.102 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-07-14T10:59:17.102 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-mako-1.0.6-13.el8.noarch 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-07-14T10:59:17.103 INFO:teuthology.orchestra.run.smithi040.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-07-14T10:59:17.104 INFO:teuthology.orchestra.run.smithi040.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-07-14T10:59:17.105 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-07-14T10:59:17.105 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-07-14T10:59:17.105 INFO:teuthology.orchestra.run.smithi040.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-portend-2.6-1.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-07-14T10:59:17.106 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pycparser-2.14-14.el8.noarch 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytz-2017.2-9.el8.noarch 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.107 INFO:teuthology.orchestra.run.smithi040.stdout: python3-routes-2.4.1-12.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rsa-4.8-1.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-07-14T10:59:17.108 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.109 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: socat-1.7.4.1-1.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: thrift-0.13.0-2.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: uuid-1.6.2-43.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T10:59:17.110 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-07-14T10:59:17.377 DEBUG:teuthology.parallel:result is None 2023-07-14T10:59:22.164 INFO:teuthology.orchestra.run.smithi130.stdout:[MIRROR] gperftools-libs-2.7-9.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirror.es.its.nyu.edu/epel/8/Everything/x86_64/Packages/g/gperftools-libs-2.7-9.el8.x86_64.rpm [Failed to connect to mirror.es.its.nyu.edu port 80: Connection timed out] 2023-07-14T10:59:22.165 INFO:teuthology.orchestra.run.smithi130.stdout:[MIRROR] liboath-2.6.2-3.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirror.es.its.nyu.edu/epel/8/Everything/x86_64/Packages/l/liboath-2.6.2-3.el8.x86_64.rpm [Failed to connect to mirror.es.its.nyu.edu port 80: Connection timed out] 2023-07-14T10:59:22.166 INFO:teuthology.orchestra.run.smithi130.stdout:[MIRROR] libunwind-1.3.1-3.el8.x86_64.rpm: Curl error (7): Couldn't connect to server for http://mirror.es.its.nyu.edu/epel/8/Everything/x86_64/Packages/l/libunwind-1.3.1-3.el8.x86_64.rpm [Failed to connect to mirror.es.its.nyu.edu port 80: Connection timed out] 2023-07-14T10:59:22.290 INFO:teuthology.orchestra.run.smithi130.stdout:(39/131): liboath-2.6.2-3.el8.x86_64.rpm 458 B/s | 59 kB 02:11 2023-07-14T10:59:22.317 INFO:teuthology.orchestra.run.smithi130.stdout:(40/131): libunwind-1.3.1-3.el8.x86_64.rpm 587 B/s | 75 kB 02:10 2023-07-14T10:59:22.368 INFO:teuthology.orchestra.run.smithi130.stdout:(41/131): gperftools-libs-2.7-9.el8.x86_64.rpm 2.3 kB/s | 306 kB 02:11 2023-07-14T10:59:22.390 INFO:teuthology.orchestra.run.smithi130.stdout:(42/131): python3-bcrypt-3.1.6-2.el8.1.x86_64.r 580 kB/s | 44 kB 00:00 2023-07-14T10:59:22.424 INFO:teuthology.orchestra.run.smithi130.stdout:(43/131): python3-asyncssh-2.7.0-2.el8.noarch.r 3.5 MB/s | 476 kB 00:00 2023-07-14T10:59:22.450 INFO:teuthology.orchestra.run.smithi130.stdout:(44/131): python3-cachetools-3.1.1-4.el8.noarch 563 kB/s | 33 kB 00:00 2023-07-14T10:59:22.467 INFO:teuthology.orchestra.run.smithi130.stdout:(45/131): python3-certifi-2018.10.15-7.el8.noar 389 kB/s | 16 kB 00:00 2023-07-14T10:59:22.492 INFO:teuthology.orchestra.run.smithi130.stdout:(46/131): python3-beautifulsoup4-4.6.3-2.el8.1. 1.4 MB/s | 185 kB 00:00 2023-07-14T10:59:22.525 INFO:teuthology.orchestra.run.smithi130.stdout:(47/131): python3-cherrypy-18.4.0-1.el8.noarch. 6.4 MB/s | 384 kB 00:00 2023-07-14T10:59:22.558 INFO:teuthology.orchestra.run.smithi130.stdout:(48/131): python3-cheroot-8.5.2-1.el8.noarch.rp 1.5 MB/s | 173 kB 00:00 2023-07-14T10:59:22.575 INFO:teuthology.orchestra.run.smithi130.stdout:(49/131): python3-google-auth-1.1.1-10.el8.noar 969 kB/s | 81 kB 00:00 2023-07-14T10:59:22.592 INFO:teuthology.orchestra.run.smithi130.stdout:(50/131): python3-jaraco-6.2-6.el8.noarch.rpm 166 kB/s | 11 kB 00:00 2023-07-14T10:59:22.617 INFO:teuthology.orchestra.run.smithi130.stdout:(51/131): python3-jaraco-functools-2.0-4.el8.no 421 kB/s | 18 kB 00:00 2023-07-14T10:59:22.634 INFO:teuthology.orchestra.run.smithi130.stdout:(52/131): python3-logutils-0.3.5-11.el8.noarch. 1.1 MB/s | 49 kB 00:00 2023-07-14T10:59:22.710 INFO:teuthology.orchestra.run.smithi130.stdout:(53/131): python3-kubernetes-11.0.0-6.el8.noarc 8.7 MB/s | 1.2 MB 00:00 2023-07-14T10:59:22.735 INFO:teuthology.orchestra.run.smithi130.stdout:(54/131): python3-more-itertools-7.2.0-3.el8.no 500 kB/s | 59 kB 00:00 2023-07-14T10:59:22.760 INFO:teuthology.orchestra.run.smithi130.stdout:(55/131): python3-natsort-7.1.1-2.el8.noarch.rp 480 kB/s | 60 kB 00:00 2023-07-14T10:59:22.785 INFO:teuthology.orchestra.run.smithi130.stdout:(56/131): python3-pecan-1.3.2-9.el8.noarch.rpm 3.7 MB/s | 283 kB 00:00 2023-07-14T10:59:22.811 INFO:teuthology.orchestra.run.smithi130.stdout:(57/131): python3-portend-2.6-1.el8.noarch.rpm 311 kB/s | 16 kB 00:00 2023-07-14T10:59:22.836 INFO:teuthology.orchestra.run.smithi130.stdout:(58/131): python3-repoze-lru-0.7-6.el8.noarch.r 447 kB/s | 34 kB 00:00 2023-07-14T10:59:22.862 INFO:teuthology.orchestra.run.smithi130.stdout:(59/131): python3-rsa-4.8-1.el8.noarch.rpm 1.2 MB/s | 61 kB 00:00 2023-07-14T10:59:22.889 INFO:teuthology.orchestra.run.smithi130.stdout:(60/131): python3-simplegeneric-0.8.1-17.el8.no 375 kB/s | 19 kB 00:00 2023-07-14T10:59:22.912 INFO:teuthology.orchestra.run.smithi130.stdout:(61/131): python3-singledispatch-3.4.0.3-18.el8 476 kB/s | 24 kB 00:00 2023-07-14T10:59:22.937 INFO:teuthology.orchestra.run.smithi130.stdout:(62/131): python3-tempora-1.14.1-5.el8.noarch.r 581 kB/s | 29 kB 00:00 2023-07-14T10:59:22.954 INFO:teuthology.orchestra.run.smithi130.stdout:(63/131): python3-routes-2.4.1-12.el8.noarch.rp 1.1 MB/s | 196 kB 00:00 2023-07-14T10:59:22.980 INFO:teuthology.orchestra.run.smithi130.stdout:(64/131): python3-trustme-0.6.0-4.el8.noarch.rp 409 kB/s | 27 kB 00:00 2023-07-14T10:59:23.005 INFO:teuthology.orchestra.run.smithi130.stdout:(65/131): python3-waitress-1.4.3-1.el8.noarch.r 3.7 MB/s | 255 kB 00:00 2023-07-14T10:59:23.030 INFO:teuthology.orchestra.run.smithi130.stdout:(66/131): python3-websocket-client-0.56.0-5.el8 1.2 MB/s | 61 kB 00:00 2023-07-14T10:59:23.057 INFO:teuthology.orchestra.run.smithi130.stdout:(67/131): python3-webtest-2.0.33-1.el8.noarch.r 1.7 MB/s | 85 kB 00:00 2023-07-14T10:59:23.082 INFO:teuthology.orchestra.run.smithi130.stdout:(68/131): python3-zc-lockfile-2.0-2.el8.noarch. 459 kB/s | 23 kB 00:00 2023-07-14T10:59:23.099 INFO:teuthology.orchestra.run.smithi130.stdout:(69/131): python3-webob-1.8.5-1.el8.1.noarch.rp 1.7 MB/s | 251 kB 00:00 2023-07-14T10:59:23.123 INFO:teuthology.orchestra.run.smithi130.stdout:(70/131): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.6 MB/s | 70 kB 00:00 2023-07-14T10:59:23.189 INFO:teuthology.orchestra.run.smithi130.stdout:(71/131): thrift-0.13.0-2.el8.x86_64.rpm 13 MB/s | 1.7 MB 00:00 2023-07-14T10:59:23.349 INFO:teuthology.orchestra.run.smithi130.stdout:(72/131): librabbitmq-0.9.0-3.el8.x86_64.rpm 187 kB/s | 47 kB 00:00 2023-07-14T10:59:23.430 INFO:teuthology.orchestra.run.smithi130.stdout:(73/131): libselinux-devel-2.9-5.el8.x86_64.rpm 677 kB/s | 200 kB 00:00 2023-07-14T10:59:23.541 INFO:teuthology.orchestra.run.smithi130.stdout:(74/131): python3-jwt-1.6.1-2.el8.noarch.rpm 123 kB/s | 43 kB 00:00 2023-07-14T10:59:23.725 INFO:teuthology.orchestra.run.smithi130.stdout:(75/131): pcre2-utf32-10.32-2.el8.x86_64.rpm 587 kB/s | 220 kB 00:00 2023-07-14T10:59:23.809 INFO:teuthology.orchestra.run.smithi130.stdout:(76/131): keyutils-libs-devel-1.5.10-9.el8.x86_ 125 kB/s | 48 kB 00:00 2023-07-14T10:59:23.892 INFO:teuthology.orchestra.run.smithi130.stdout:(77/131): libsepol-devel-2.9-3.el8.x86_64.rpm 248 kB/s | 87 kB 00:00 2023-07-14T10:59:23.943 INFO:teuthology.orchestra.run.smithi130.stdout:(78/131): python3-requests-oauthlib-1.0.0-1.el8 196 kB/s | 43 kB 00:00 2023-07-14T10:59:24.001 INFO:teuthology.orchestra.run.smithi130.stdout:(79/131): python3-pycparser-2.14-14.el8.noarch. 567 kB/s | 109 kB 00:00 2023-07-14T10:59:24.109 INFO:teuthology.orchestra.run.smithi130.stdout:(80/131): pcre2-utf16-10.32-2.el8.x86_64.rpm 1.0 MB/s | 229 kB 00:00 2023-07-14T10:59:24.153 INFO:teuthology.orchestra.run.smithi130.stdout:(81/131): krb5-devel-1.18.2-14.el8.x86_64.rpm 2.6 MB/s | 560 kB 00:00 2023-07-14T10:59:24.285 INFO:teuthology.orchestra.run.smithi130.stdout:(82/131): libaio-devel-0.3.112-1.el8.x86_64.rpm 66 kB/s | 19 kB 00:00 2023-07-14T10:59:24.319 INFO:teuthology.orchestra.run.smithi130.stdout:(83/131): python3-oauthlib-2.1.0-1.el8.noarch.r 741 kB/s | 155 kB 00:00 2023-07-14T10:59:24.382 INFO:teuthology.orchestra.run.smithi130.stdout:(84/131): python3-cryptography-3.2.1-5.el8.x86_ 2.4 MB/s | 559 kB 00:00 2023-07-14T10:59:24.554 INFO:teuthology.orchestra.run.smithi130.stdout:(85/131): libverto-devel-0.3.0-5.el8.x86_64.rpm 67 kB/s | 18 kB 00:00 2023-07-14T10:59:24.579 INFO:teuthology.orchestra.run.smithi130.stdout:(86/131): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.2 MB/s | 238 kB 00:00 2023-07-14T10:59:24.612 INFO:teuthology.orchestra.run.smithi130.stdout:(87/131): pcre2-devel-10.32-2.el8.x86_64.rpm 2.0 MB/s | 605 kB 00:00 2023-07-14T10:59:24.787 INFO:teuthology.orchestra.run.smithi130.stdout:(88/131): xfsprogs-devel-5.0.0-10.el8.x86_64.rp 347 kB/s | 72 kB 00:00 2023-07-14T10:59:24.847 INFO:teuthology.orchestra.run.smithi130.stdout:(89/131): openssl-devel-1.1.1k-7.el8_6.x86_64.r 7.9 MB/s | 2.3 MB 00:00 2023-07-14T10:59:24.887 INFO:teuthology.orchestra.run.smithi130.stdout:(90/131): fuse-2.9.7-15.el8.x86_64.rpm 300 kB/s | 83 kB 00:00 2023-07-14T10:59:24.963 INFO:teuthology.orchestra.run.smithi130.stdout:(91/131): elfutils-libelf-devel-0.186-1.el8.x86 341 kB/s | 60 kB 00:00 2023-07-14T10:59:24.988 INFO:teuthology.orchestra.run.smithi130.stdout:(92/131): zlib-devel-1.2.11-18.el8_5.x86_64.rpm 574 kB/s | 58 kB 00:00 2023-07-14T10:59:25.138 INFO:teuthology.orchestra.run.smithi130.stdout:(93/131): libuuid-devel-2.32.1-35.el8.x86_64.rp 335 kB/s | 98 kB 00:00 2023-07-14T10:59:25.375 INFO:teuthology.orchestra.run.smithi130.stdout:(94/131): libquadmath-8.5.0-10.1.el8_6.x86_64.r 733 kB/s | 171 kB 00:00 2023-07-14T10:59:25.399 INFO:teuthology.orchestra.run.smithi130.stdout:(95/131): libcom_err-devel-1.45.6-4.el8.x86_64. 89 kB/s | 39 kB 00:00 2023-07-14T10:59:25.431 INFO:teuthology.orchestra.run.smithi130.stdout:(96/131): libgfortran-8.5.0-10.1.el8_6.x86_64.r 1.4 MB/s | 644 kB 00:00 2023-07-14T10:59:25.724 INFO:teuthology.orchestra.run.smithi130.stdout:(97/131): python3-scipy-1.0.0-21.module+el8.5.0 42 MB/s | 14 MB 00:00 2023-07-14T10:59:25.773 INFO:teuthology.orchestra.run.smithi130.stdout:(98/131): python3-pyasn1-modules-0.3.7-6.el8.no 322 kB/s | 110 kB 00:00 2023-07-14T10:59:25.799 INFO:teuthology.orchestra.run.smithi130.stdout:(99/131): NetworkManager-initscripts-updown-1.3 323 kB/s | 138 kB 00:00 2023-07-14T10:59:25.865 INFO:teuthology.orchestra.run.smithi130.stdout:(100/131): python3-pytz-2017.2-9.el8.noarch.rpm 381 kB/s | 54 kB 00:00 2023-07-14T10:59:25.916 INFO:teuthology.orchestra.run.smithi130.stdout:(101/131): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.8 MB/s | 259 kB 00:00 2023-07-14T10:59:25.958 INFO:teuthology.orchestra.run.smithi130.stdout:(102/131): socat-1.7.4.1-1.el8.x86_64.rpm 2.0 MB/s | 323 kB 00:00 2023-07-14T10:59:25.985 INFO:teuthology.orchestra.run.smithi130.stdout:(103/131): python3-markupsafe-0.23-19.el8.x86_6 331 kB/s | 39 kB 00:00 2023-07-14T10:59:26.046 INFO:teuthology.orchestra.run.smithi130.stdout:(104/131): bison-3.0.4-10.el8.x86_64.rpm 5.4 MB/s | 688 kB 00:00 2023-07-14T10:59:26.075 INFO:teuthology.orchestra.run.smithi130.stdout:(105/131): python3-jinja2-2.10.1-3.el8.noarch.r 4.5 MB/s | 538 kB 00:00 2023-07-14T10:59:26.126 INFO:teuthology.orchestra.run.smithi130.stdout:(106/131): python3-mako-1.0.6-13.el8.noarch.rpm 1.1 MB/s | 157 kB 00:00 2023-07-14T10:59:26.184 INFO:teuthology.orchestra.run.smithi130.stdout:(107/131): oniguruma-6.8.2-2.el8.x86_64.rpm 1.3 MB/s | 187 kB 00:00 2023-07-14T10:59:26.234 INFO:teuthology.orchestra.run.smithi130.stdout:(108/131): automake-1.16.1-7.el8.noarch.rpm 4.4 MB/s | 713 kB 00:00 2023-07-14T10:59:26.293 INFO:teuthology.orchestra.run.smithi130.stdout:(109/131): python3-jsonpatch-1.21-2.el8.noarch. 244 kB/s | 27 kB 00:00 2023-07-14T10:59:26.405 INFO:teuthology.orchestra.run.smithi130.stdout:(110/131): python3-babel-2.5.1-7.el8.noarch.rpm 17 MB/s | 4.8 MB 00:00 2023-07-14T10:59:26.419 INFO:teuthology.orchestra.run.smithi130.stdout:(111/131): python3-prettytable-0.7.2-14.el8.noa 351 kB/s | 44 kB 00:00 2023-07-14T10:59:26.453 INFO:teuthology.orchestra.run.smithi130.stdout:(112/131): flex-2.6.1-9.el8.x86_64.rpm 1.4 MB/s | 320 kB 00:00 2023-07-14T10:59:26.644 INFO:teuthology.orchestra.run.smithi130.stdout:(113/131): python3-werkzeug-0.12.2-4.el8.noarch 2.0 MB/s | 457 kB 00:00 2023-07-14T10:59:26.670 INFO:teuthology.orchestra.run.smithi130.stdout:(114/131): python3-numpy-f2py-1.14.3-10.el8.x86 1.0 MB/s | 225 kB 00:00 2023-07-14T10:59:26.753 INFO:teuthology.orchestra.run.smithi130.stdout:(115/131): python3-numpy-1.14.3-10.el8.x86_64.r 11 MB/s | 3.7 MB 00:00 2023-07-14T10:59:26.828 INFO:teuthology.orchestra.run.smithi130.stdout:(116/131): python3-pyasn1-0.3.7-6.el8.noarch.rp 797 kB/s | 126 kB 00:00 2023-07-14T10:59:26.854 INFO:teuthology.orchestra.run.smithi130.stdout:(117/131): python3-jsonpointer-1.10-11.el8.noar 200 kB/s | 20 kB 00:00 2023-07-14T10:59:26.990 INFO:teuthology.orchestra.run.smithi130.stdout:(118/131): python3-pyOpenSSL-19.0.0-1.el8.noarc 652 kB/s | 103 kB 00:00 2023-07-14T10:59:27.021 INFO:teuthology.orchestra.run.smithi130.stdout:(119/131): libtool-2.4.6-25.el8.x86_64.rpm 4.1 MB/s | 709 kB 00:00 2023-07-14T10:59:27.055 INFO:teuthology.orchestra.run.smithi130.stdout:(120/131): uuid-1.6.2-43.el8.x86_64.rpm 156 kB/s | 64 kB 00:00 2023-07-14T10:59:27.180 INFO:teuthology.orchestra.run.smithi130.stdout:(121/131): libpq-13.5-1.el8.x86_64.rpm 1.2 MB/s | 198 kB 00:00 2023-07-14T10:59:27.309 INFO:teuthology.orchestra.run.smithi130.stdout:(122/131): openblas-threads-0.3.15-3.el8.x86_64 15 MB/s | 4.9 MB 00:00 2023-07-14T10:59:27.347 INFO:teuthology.orchestra.run.smithi130.stdout:(123/131): jq-1.6-3.el8.x86_64.rpm 1.2 MB/s | 202 kB 00:00 2023-07-14T10:59:27.414 INFO:teuthology.orchestra.run.smithi130.stdout:(124/131): openblas-0.3.15-3.el8.x86_64.rpm 13 MB/s | 4.7 MB 00:00 2023-07-14T10:59:27.445 INFO:teuthology.orchestra.run.smithi130.stdout:(125/131): librdkafka-0.11.4-3.el8.x86_64.rpm 2.7 MB/s | 354 kB 00:00 2023-07-14T10:59:27.524 INFO:teuthology.orchestra.run.smithi130.stdout:(126/131): postgresql-10.21-2.module+el8.6.0+15 18 MB/s | 1.5 MB 00:00 2023-07-14T10:59:27.575 INFO:teuthology.orchestra.run.smithi130.stdout:(127/131): postgresql-contrib-10.21-2.module+el 5.0 MB/s | 808 kB 00:00 2023-07-14T10:59:27.689 INFO:teuthology.orchestra.run.smithi130.stdout:(128/131): postgresql-server-10.21-2.module+el8 16 MB/s | 5.1 MB 00:00 2023-07-14T10:59:27.900 INFO:teuthology.orchestra.run.smithi130.stdout:(129/131): iproute-5.15.0-4.el8_6.1.x86_64.rpm 3.5 MB/s | 798 kB 00:00 2023-07-14T10:59:28.201 INFO:teuthology.orchestra.run.smithi130.stdout:(130/131): librados2-18.0.0-4757.g3fc0089c.el8. 5.4 MB/s | 3.7 MB 00:00 2023-07-14T10:59:28.276 INFO:teuthology.orchestra.run.smithi130.stdout:(131/131): librbd1-18.0.0-4757.g3fc0089c.el8.x8 5.1 MB/s | 3.6 MB 00:00 2023-07-14T10:59:28.277 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2023-07-14T10:59:28.277 INFO:teuthology.orchestra.run.smithi130.stdout:Total 1.4 MB/s | 205 MB 02:23 2023-07-14T10:59:29.062 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2023-07-14T10:59:29.181 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2023-07-14T10:59:29.181 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2023-07-14T10:59:30.305 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2023-07-14T10:59:30.306 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2023-07-14T10:59:35.196 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2023-07-14T10:59:35.365 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : thrift-0.13.0-2.el8.x86_64 1/134 2023-07-14T10:59:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/134 2023-07-14T10:59:35.718 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 3/134 2023-07-14T10:59:35.814 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 3/134 2023-07-14T10:59:36.084 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-07-14T10:59:36.471 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 4/134 2023-07-14T10:59:36.701 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 5/134 2023-07-14T10:59:37.090 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 5/134 2023-07-14T10:59:37.277 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:59:37.399 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T10:59:37.617 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:59:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T10:59:37.873 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T10:59:38.040 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 9/134 2023-07-14T10:59:38.296 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libpq-13.5-1.el8.x86_64 10/134 2023-07-14T10:59:38.495 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 11/134 2023-07-14T10:59:38.939 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 12/134 2023-07-14T10:59:39.204 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-10.21-2.module+el8.6.0+15342+53518fac 13/134 2023-07-14T10:59:39.319 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:59:39.419 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 14/134 2023-07-14T10:59:39.559 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:59:39.704 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 15/134 2023-07-14T10:59:39.913 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 16/134 2023-07-14T10:59:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 17/134 2023-07-14T10:59:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 18/134 2023-07-14T10:59:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-07-14T10:59:40.513 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 19/134 2023-07-14T10:59:40.667 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-07-14T10:59:41.311 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 20/134 2023-07-14T10:59:41.475 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-07-14T10:59:42.063 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 21/134 2023-07-14T10:59:42.283 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-0.3.15-3.el8.x86_64 22/134 2023-07-14T10:59:42.970 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 22/134 2023-07-14T10:59:43.216 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 23/134 2023-07-14T10:59:43.422 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : zlib-devel-1.2.11-18.el8_5.x86_64 24/134 2023-07-14T10:59:43.659 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 25/134 2023-07-14T10:59:43.857 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 26/134 2023-07-14T10:59:43.986 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 27/134 2023-07-14T10:59:44.195 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 28/134 2023-07-14T10:59:44.400 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-07-14T10:59:46.155 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 29/134 2023-07-14T10:59:47.218 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 30/134 2023-07-14T10:59:47.555 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 31/134 2023-07-14T10:59:47.822 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 32/134 2023-07-14T10:59:48.072 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 33/134 2023-07-14T10:59:48.269 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rsa-4.8-1.el8.noarch 34/134 2023-07-14T10:59:48.485 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 35/134 2023-07-14T10:59:48.655 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 36/134 2023-07-14T10:59:48.816 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 37/134 2023-07-14T10:59:48.989 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 38/134 2023-07-14T10:59:49.148 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 39/134 2023-07-14T10:59:49.332 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : uuid-1.6.2-43.el8.x86_64 40/134 2023-07-14T10:59:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: uuid-1.6.2-43.el8.x86_64 40/134 2023-07-14T10:59:49.635 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 41/134 2023-07-14T10:59:49.836 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : automake-1.16.1-7.el8.noarch 42/134 2023-07-14T10:59:50.007 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 43/134 2023-07-14T10:59:50.067 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 43/134 2023-07-14T10:59:50.278 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : jq-1.6-3.el8.x86_64 44/134 2023-07-14T10:59:50.427 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 45/134 2023-07-14T10:59:50.614 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcom_err-devel-1.45.6-4.el8.x86_64 46/134 2023-07-14T10:59:50.792 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libuuid-devel-2.32.1-35.el8.x86_64 47/134 2023-07-14T10:59:50.959 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : fuse-2.9.7-15.el8.x86_64 48/134 2023-07-14T10:59:51.145 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libverto-devel-0.3.0-5.el8.x86_64 49/134 2023-07-14T10:59:51.347 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : pcre2-utf16-10.32-2.el8.x86_64 50/134 2023-07-14T10:59:51.585 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 51/134 2023-07-14T10:59:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 52/134 2023-07-14T10:59:52.311 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 53/134 2023-07-14T10:59:52.540 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 54/134 2023-07-14T10:59:52.853 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 55/134 2023-07-14T10:59:53.387 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 56/134 2023-07-14T10:59:53.744 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 57/134 2023-07-14T10:59:53.956 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 58/134 2023-07-14T10:59:54.165 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 59/134 2023-07-14T10:59:54.403 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 60/134 2023-07-14T10:59:54.568 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libsepol-devel-2.9-3.el8.x86_64 61/134 2023-07-14T10:59:54.759 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : keyutils-libs-devel-1.5.10-9.el8.x86_64 62/134 2023-07-14T10:59:55.071 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : pcre2-utf32-10.32-2.el8.x86_64 63/134 2023-07-14T10:59:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : pcre2-devel-10.32-2.el8.x86_64 64/134 2023-07-14T10:59:55.569 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libselinux-devel-2.9-5.el8.x86_64 65/134 2023-07-14T10:59:55.799 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : krb5-devel-1.18.2-14.el8.x86_64 66/134 2023-07-14T10:59:55.972 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 67/134 2023-07-14T10:59:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 68/134 2023-07-14T10:59:56.450 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 69/134 2023-07-14T10:59:56.713 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 70/134 2023-07-14T10:59:56.844 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 71/134 2023-07-14T10:59:56.992 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 72/134 2023-07-14T10:59:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 73/134 2023-07-14T10:59:57.403 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 74/134 2023-07-14T10:59:57.585 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 75/134 2023-07-14T10:59:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 76/134 2023-07-14T10:59:57.920 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 77/134 2023-07-14T10:59:58.128 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 78/134 2023-07-14T10:59:58.394 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 79/134 2023-07-14T10:59:58.604 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 80/134 2023-07-14T10:59:58.878 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-portend-2.6-1.el8.noarch 81/134 2023-07-14T10:59:59.080 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 82/134 2023-07-14T10:59:59.221 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 83/134 2023-07-14T10:59:59.457 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 84/134 2023-07-14T11:00:00.544 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 85/134 2023-07-14T11:00:00.842 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 86/134 2023-07-14T11:00:01.097 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 87/134 2023-07-14T11:00:01.433 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 88/134 2023-07-14T11:00:01.699 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 89/134 2023-07-14T11:00:01.915 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 90/134 2023-07-14T11:00:02.688 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 91/134 2023-07-14T11:00:02.905 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T11:00:02.979 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 92/134 2023-07-14T11:00:03.147 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 93/134 2023-07-14T11:00:06.859 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T11:00:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T11:00:07.707 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 94/134 2023-07-14T11:00:07.992 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T11:00:08.089 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 95/134 2023-07-14T11:00:08.221 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T11:00:27.787 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 96/134 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /sys 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /proc 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /dev 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /run 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /mnt 2023-07-14T11:00:27.788 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /var/tmp 2023-07-14T11:00:27.789 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /home 2023-07-14T11:00:27.789 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /tmp 2023-07-14T11:00:27.789 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /dev 2023-07-14T11:00:27.789 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:00:28.168 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T11:00:29.730 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 97/134 2023-07-14T11:00:29.898 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T11:00:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 98/134 2023-07-14T11:00:30.326 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T11:00:30.779 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 99/134 2023-07-14T11:00:30.926 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T11:00:30.943 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 100/134 2023-07-14T11:00:32.663 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T11:00:32.885 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 101/134 2023-07-14T11:00:32.998 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 102/134 2023-07-14T11:00:33.175 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 103/134 2023-07-14T11:00:33.392 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T11:00:33.607 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 104/134 2023-07-14T11:00:33.897 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T11:00:34.859 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 105/134 2023-07-14T11:00:35.042 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T11:00:35.131 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 106/134 2023-07-14T11:00:35.517 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 107/134 2023-07-14T11:00:35.723 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T11:00:35.757 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 108/134 2023-07-14T11:00:35.975 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T11:00:35.988 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 109/134 2023-07-14T11:00:37.369 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 110/134 2023-07-14T11:00:37.570 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T11:00:37.607 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 111/134 2023-07-14T11:00:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T11:00:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 112/134 2023-07-14T11:00:38.000 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T11:00:38.366 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 113/134 2023-07-14T11:00:38.542 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T11:00:43.196 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 114/134 2023-07-14T11:00:44.062 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 115/134 2023-07-14T11:00:44.356 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openssl-devel-1:1.1.1k-7.el8_6.x86_64 116/134 2023-07-14T11:00:44.585 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 117/134 2023-07-14T11:00:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : xfsprogs-devel-5.0.0-10.el8.x86_64 118/134 2023-07-14T11:00:44.990 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libtool-2.4.6-25.el8.x86_64 119/134 2023-07-14T11:00:45.188 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libtool-2.4.6-25.el8.x86_64 119/134 2023-07-14T11:00:45.420 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 120/134 2023-07-14T11:00:45.636 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 121/134 2023-07-14T11:00:45.766 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : elfutils-libelf-devel-0.186-1.el8.x86_64 122/134 2023-07-14T11:00:48.031 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T11:00:48.315 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T11:00:48.334 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: postgresql-server-10.21-2.module+el8.6.0+15342+5 123/134 2023-07-14T11:00:48.507 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 124/134 2023-07-14T11:00:48.823 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 125/134 2023-07-14T11:00:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 126/134 2023-07-14T11:00:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : iproute-5.15.0-4.el8_6.1.x86_64 127/134 2023-07-14T11:00:49.639 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flex-2.6.1-9.el8.x86_64 128/134 2023-07-14T11:00:49.722 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: flex-2.6.1-9.el8.x86_64 128/134 2023-07-14T11:00:49.957 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : bison-3.0.4-10.el8.x86_64 129/134 2023-07-14T11:00:49.969 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: bison-3.0.4-10.el8.x86_64 129/134 2023-07-14T11:00:50.123 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-07-14T11:00:50.130 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: NetworkManager-initscripts-updown-1:1.36.0-9.el8 130/134 2023-07-14T11:00:50.292 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libaio-devel-0.3.112-1.el8.x86_64 131/134 2023-07-14T11:00:50.292 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T11:00:50.468 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T11:00:50.468 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T11:00:50.685 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 133/134 2023-07-14T11:00:50.930 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T11:00:51.445 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T11:00:51.445 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 1/134 2023-07-14T11:00:51.445 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2/134 2023-07-14T11:00:51.446 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 3/134 2023-07-14T11:00:51.446 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 4/134 2023-07-14T11:00:51.446 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-immutable-object-cache-2:18.0.0-4757.g3fc00 5/134 2023-07-14T11:00:51.446 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 6/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 7/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 8/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 9/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 10/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 11/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 12/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 13/134 2023-07-14T11:00:51.447 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_ 14/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 15/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 16/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_6 17/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86 18/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 19/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el 20/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8. 21/134 2023-07-14T11:00:51.448 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_6 22/134 2023-07-14T11:00:51.449 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 23/134 2023-07-14T11:00:51.449 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 24/134 2023-07-14T11:00:51.449 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 25/134 2023-07-14T11:00:51.450 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 26/134 2023-07-14T11:00:51.450 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 27/134 2023-07-14T11:00:51.470 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 28/134 2023-07-14T11:00:51.471 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c. 29/134 2023-07-14T11:00:51.472 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noa 30/134 2023-07-14T11:00:51.472 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.n 31/134 2023-07-14T11:00:51.472 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc 32/134 2023-07-14T11:00:51.474 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el 33/134 2023-07-14T11:00:51.474 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 34/134 2023-07-14T11:00:51.474 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.e 35/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 36/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 37/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 38/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 39/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 40/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 41/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 42/134 2023-07-14T11:00:51.475 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 43/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 44/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 45/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 46/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 47/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 48/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 49/134 2023-07-14T11:00:51.476 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 50/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 51/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 52/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 53/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 54/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 55/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 56/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-portend-2.6-1.el8.noarch 57/134 2023-07-14T11:00:51.477 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 58/134 2023-07-14T11:00:51.478 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 59/134 2023-07-14T11:00:51.478 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rsa-4.8-1.el8.noarch 60/134 2023-07-14T11:00:51.478 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 61/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 62/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 63/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 64/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 65/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 66/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 67/134 2023-07-14T11:00:51.479 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 68/134 2023-07-14T11:00:51.480 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 69/134 2023-07-14T11:00:51.480 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 70/134 2023-07-14T11:00:51.480 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 71/134 2023-07-14T11:00:51.480 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 72/134 2023-07-14T11:00:51.480 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libselinux-devel-2.9-5.el8.x86_64 73/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 74/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : pcre2-utf32-10.32-2.el8.x86_64 75/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : keyutils-libs-devel-1.5.10-9.el8.x86_64 76/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libsepol-devel-2.9-3.el8.x86_64 77/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 78/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 79/134 2023-07-14T11:00:51.481 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : pcre2-utf16-10.32-2.el8.x86_64 80/134 2023-07-14T11:00:51.482 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : krb5-devel-1.18.2-14.el8.x86_64 81/134 2023-07-14T11:00:51.482 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libaio-devel-0.3.112-1.el8.x86_64 82/134 2023-07-14T11:00:51.483 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 83/134 2023-07-14T11:00:51.483 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 84/134 2023-07-14T11:00:51.483 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libverto-devel-0.3.0-5.el8.x86_64 85/134 2023-07-14T11:00:51.483 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : pcre2-devel-10.32-2.el8.x86_64 86/134 2023-07-14T11:00:51.485 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 87/134 2023-07-14T11:00:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openssl-devel-1:1.1.1k-7.el8_6.x86_64 88/134 2023-07-14T11:00:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : xfsprogs-devel-5.0.0-10.el8.x86_64 89/134 2023-07-14T11:00:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 90/134 2023-07-14T11:00:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : elfutils-libelf-devel-0.186-1.el8.x86_64 91/134 2023-07-14T11:00:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libuuid-devel-2.32.1-35.el8.x86_64 92/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : zlib-devel-1.2.11-18.el8_5.x86_64 93/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcom_err-devel-1.45.6-4.el8.x86_64 94/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 95/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 96/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : NetworkManager-initscripts-updown-1:1.36.0-9.el8 97/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd 98/134 2023-07-14T11:00:51.487 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 99/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 100/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 101/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 102/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 103/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : bison-3.0.4-10.el8.x86_64 104/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 105/134 2023-07-14T11:00:51.488 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 106/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 107/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : automake-1.16.1-7.el8.noarch 108/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 109/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 110/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flex-2.6.1-9.el8.x86_64 111/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 112/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 113/134 2023-07-14T11:00:51.489 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 114/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 115/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : uuid-1.6.2-43.el8.x86_64 116/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 117/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 118/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 119/134 2023-07-14T11:00:51.490 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libtool-2.4.6-25.el8.x86_64 120/134 2023-07-14T11:00:51.491 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 121/134 2023-07-14T11:00:51.491 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libpq-13.5-1.el8.x86_64 122/134 2023-07-14T11:00:51.491 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 123/134 2023-07-14T11:00:51.491 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : jq-1.6-3.el8.x86_64 124/134 2023-07-14T11:00:51.491 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 125/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-server-10.21-2.module+el8.6.0+15342+5 126/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-contrib-10.21-2.module+el8.6.0+15342+ 127/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : postgresql-10.21-2.module+el8.6.0+15342+53518fac 128/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 129/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 130/134 2023-07-14T11:00:51.492 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 131/134 2023-07-14T11:00:51.493 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 132/134 2023-07-14T11:00:51.493 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : iproute-5.15.0-4.el8_6.1.x86_64 133/134 2023-07-14T11:00:53.907 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : iproute-5.15.0-4.el8.x86_64 134/134 2023-07-14T11:00:53.908 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:04:01 ago on Fri 14 Jul 2023 10:56:52 AM UTC. 2023-07-14T11:01:03.526 INFO:teuthology.orchestra.run.smithi130.stdout:Installed products updated. 2023-07-14T11:01:03.739 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout:Upgraded: 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout: iproute-5.15.0-4.el8_6.1.x86_64 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout: librados2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2023-07-14T11:01:03.740 INFO:teuthology.orchestra.run.smithi130.stdout: NetworkManager-initscripts-updown-1:1.36.0-9.el8_6.noarch 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: automake-1.16.1-7.el8.noarch 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: bison-3.0.4-10.el8.x86_64 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.741 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.742 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.743 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-volume-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: cephfs-top-2:18.0.0-4757.g3fc0089c.el8.noarch 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: elfutils-libelf-devel-0.186-1.el8.x86_64 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: flex-2.6.1-9.el8.x86_64 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: fuse-2.9.7-15.el8.x86_64 2023-07-14T11:01:03.744 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: jq-1.6-3.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: keyutils-libs-devel-1.5.10-9.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: krb5-devel-1.18.2-14.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: libaio-devel-0.3.112-1.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.745 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: libcom_err-devel-1.45.6-4.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: liboath-2.6.2-3.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: libpq-13.5-1.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.746 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.747 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-07-14T11:01:03.747 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.747 INFO:teuthology.orchestra.run.smithi130.stdout: libselinux-devel-2.9-5.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: libsepol-devel-2.9-3.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: libtool-2.4.6-25.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind-1.3.1-3.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: libuuid-devel-2.32.1-35.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: libverto-devel-0.3.0-5.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-07-14T11:01:03.748 INFO:teuthology.orchestra.run.smithi130.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-0.3.15-3.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: openssl-devel-1:1.1.1k-7.el8_6.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-devel-10.32-2.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-utf16-10.32-2.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: pcre2-utf32-10.32-2.el8.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T11:01:03.749 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-contrib-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: postgresql-server-10.21-2.module+el8.6.0+15342+53518fac.x86_64 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel-2.5.1-7.el8.noarch 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.750 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-07-14T11:01:03.751 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-6.2-6.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-07-14T11:01:03.752 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako-1.0.6-13.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-07-14T11:01:03.753 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-07-14T11:01:03.754 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-07-14T11:01:03.754 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-07-14T11:01:03.754 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend-2.6-1.el8.noarch 2023-07-14T11:01:03.754 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser-2.14-14.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz-2017.2-9.el8.noarch 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.755 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes-2.4.1-12.el8.noarch 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa-4.8-1.el8.noarch 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-07-14T11:01:03.756 INFO:teuthology.orchestra.run.smithi130.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-07-14T11:01:03.757 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd-2:18.0.0-4757.g3fc0089c.el8.x86_64 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: socat-1.7.4.1-1.el8.x86_64 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: thrift-0.13.0-2.el8.x86_64 2023-07-14T11:01:03.758 INFO:teuthology.orchestra.run.smithi130.stdout: uuid-1.6.2-43.el8.x86_64 2023-07-14T11:01:03.759 INFO:teuthology.orchestra.run.smithi130.stdout: xfsprogs-devel-5.0.0-10.el8.x86_64 2023-07-14T11:01:03.759 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-07-14T11:01:03.759 INFO:teuthology.orchestra.run.smithi130.stdout: zlib-devel-1.2.11-18.el8_5.x86_64 2023-07-14T11:01:03.759 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:01:03.760 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2023-07-14T11:01:05.269 DEBUG:teuthology.parallel:result is None 2023-07-14T11:01:05.271 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:05.497 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-07-14T11:01:05.559 INFO:teuthology.orchestra.run.smithi040.stdout:18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:05.560 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:05.560 INFO:teuthology.task.install:The correct ceph version 18.0.0-4757.g3fc0089c is installed. 2023-07-14T11:01:05.561 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:05.787 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-07-14T11:01:05.818 INFO:teuthology.orchestra.run.smithi045.stdout:18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:05.819 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:05.819 INFO:teuthology.task.install:The correct ceph version 18.0.0-4757.g3fc0089c is installed. 2023-07-14T11:01:05.820 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:06.051 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-07-14T11:01:06.079 INFO:teuthology.orchestra.run.smithi130.stdout:18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:06.080 INFO:teuthology.packaging:The installed version of ceph is 18.0.0-4757.g3fc0089c.el8 2023-07-14T11:01:06.080 INFO:teuthology.task.install:The correct ceph version 18.0.0-4757.g3fc0089c is installed. 2023-07-14T11:01:06.081 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-07-14T11:01:06.082 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:01:06.082 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-07-14T11:01:06.181 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:01:06.182 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-07-14T11:01:06.230 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:01:06.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-07-14T11:01:06.277 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-07-14T11:01:06.277 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:01:06.278 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/daemon-helper 2023-07-14T11:01:06.348 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-07-14T11:01:06.457 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:01:06.458 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/daemon-helper 2023-07-14T11:01:06.491 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-07-14T11:01:06.563 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:01:06.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/daemon-helper 2023-07-14T11:01:06.595 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-07-14T11:01:06.668 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-07-14T11:01:06.669 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:01:06.669 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/adjust-ulimits 2023-07-14T11:01:06.739 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-07-14T11:01:06.848 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:01:06.848 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/adjust-ulimits 2023-07-14T11:01:06.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-07-14T11:01:06.955 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:01:06.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/adjust-ulimits 2023-07-14T11:01:06.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-07-14T11:01:07.057 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-07-14T11:01:07.057 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:01:07.058 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/stdin-killer 2023-07-14T11:01:07.126 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-07-14T11:01:07.235 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:01:07.235 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/usr/bin/stdin-killer 2023-07-14T11:01:07.268 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-07-14T11:01:07.339 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:01:07.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/stdin-killer 2023-07-14T11:01:07.373 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-07-14T11:01:07.445 INFO:teuthology.run_tasks:Running task cephadm... 2023-07-14T11:01:07.561 INFO:tasks.cephadm:Config: {'roleless': False, 'cephfs': {'ec_profile': ['m=2', 'k=2', 'crush-failure-domain=osd'], 'max_mds': 5, 'session_timeout': 300, 'standby_replay': True}, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'ms die on skipped message': False}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds bal fragment size max': 10000, 'mds bal merge size': 5, 'mds bal split bits': 3, 'mds bal split size': 100, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'cephsqlite lock renewal timeout': 900000, 'debug cephsqlite': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': 20, 'debug bluestore': 20, 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': 10, 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180, 'osd shutdown pgref assert': True, 'osd_max_omap_entries_per_request': 10000}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'overall HEALTH_', '\\(FS_DEGRADED\\)', '\\(MDS_FAILED\\)', '\\(MDS_DEGRADED\\)', '\\(FS_WITH_FAILED_MDS\\)', '\\(MDS_DAMAGE\\)', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(FS_INLINE_DATA_DEPRECATED\\)', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'slow metadata IO', 'SLOW_OPS', 'slow request'], 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b'} 2023-07-14T11:01:07.562 INFO:tasks.cephadm:Cluster image is quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:07.574 INFO:tasks.cephadm:Cluster fsid is bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:01:07.575 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-07-14T11:01:07.576 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.40', 'mon.b': '172.21.15.45', 'mon.c': '172.21.15.130'} 2023-07-14T11:01:07.576 INFO:tasks.cephadm:First mon is mon.a on smithi040 2023-07-14T11:01:07.577 INFO:tasks.cephadm:First mgr is x 2023-07-14T11:01:07.577 INFO:tasks.cephadm:Normalizing hostnames... 2023-07-14T11:01:07.578 DEBUG:teuthology.orchestra.run.smithi040:> sudo hostname $(hostname -s) 2023-07-14T11:01:07.655 DEBUG:teuthology.orchestra.run.smithi045:> sudo hostname $(hostname -s) 2023-07-14T11:01:07.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2023-07-14T11:01:07.723 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra 2023-07-14T11:01:07.724 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:07.957 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '3fc0089cc1082313a98853d48b892acdc1385b8b', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic/71399/', 'root_build_cause': 'SCMTRIGGER', 'version': '18.0.0-4757-g3fc0089c', 'node_name': '172.21.2.8+braggi08', 'job_name': 'ceph-dev-new-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos8,DIST=centos8,MACHINE_SIZE=gigantic', 'package_manager_version': '18.0.0-4757.g3fc0089c'}, 'url': 'https://1.chacra.ceph.com/r/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/', 'distro_codename': None, 'modified': '2023-07-13 19:47:46.444052', 'distro_version': '8', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-rishabh-2023Jul13', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/flavors/default/', 'archs': ['source', 'x86_64'], 'distro': 'centos'}] 2023-07-14T11:01:08.190 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref wip-rishabh-2023Jul13, sha1 3fc0089cc1082313a98853d48b892acdc1385b8b from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F8%2Fx86_64&flavor=default&sha1=3fc0089cc1082313a98853d48b892acdc1385b8b 2023-07-14T11:01:08.190 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/x86_64/flavors/default/cephadm 2023-07-14T11:01:08.191 DEBUG:teuthology.orchestra.run.smithi040:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:09.201 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-rw-r--. 1 ubuntu ubuntu 663791 Jul 14 11:01 /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:09.202 DEBUG:teuthology.orchestra.run.smithi045:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:10.078 INFO:teuthology.orchestra.run.smithi045.stdout:-rw-rw-r--. 1 ubuntu ubuntu 663791 Jul 14 11:01 /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:10.078 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/wip-rishabh-2023Jul13/3fc0089cc1082313a98853d48b892acdc1385b8b/centos/8/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:11.177 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-rw-r--. 1 ubuntu ubuntu 663791 Jul 14 11:01 /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:11.178 DEBUG:teuthology.orchestra.run.smithi040:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:11.230 DEBUG:teuthology.orchestra.run.smithi045:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:11.253 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-07-14T11:01:11.301 INFO:tasks.cephadm:Pulling image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b on all hosts... 2023-07-14T11:01:11.301 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b pull 2023-07-14T11:01:11.306 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b pull 2023-07-14T11:01:11.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b pull 2023-07-14T11:01:12.682 INFO:teuthology.orchestra.run.smithi045.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b... 2023-07-14T11:01:12.742 INFO:teuthology.orchestra.run.smithi040.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b... 2023-07-14T11:01:14.385 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b... 2023-07-14T11:01:56.837 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev)", 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "afd12e232ec04c7e77932076425916985d97c3486288fff2ba931630e84a9295", 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c4bf3497f7cb9cae2d0db734c6fa7ef173cd8dc715907bde16e4a68082d82e1d" 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2023-07-14T11:01:56.838 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-07-14T11:01:56.868 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:01:56.869 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_version": "ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev)", 2023-07-14T11:01:56.869 INFO:teuthology.orchestra.run.smithi040.stdout: "image_id": "afd12e232ec04c7e77932076425916985d97c3486288fff2ba931630e84a9295", 2023-07-14T11:01:56.869 INFO:teuthology.orchestra.run.smithi040.stdout: "repo_digests": [ 2023-07-14T11:01:56.869 INFO:teuthology.orchestra.run.smithi040.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c4bf3497f7cb9cae2d0db734c6fa7ef173cd8dc715907bde16e4a68082d82e1d" 2023-07-14T11:01:56.869 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:01:56.870 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:01:58.827 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_version": "ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev)", 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout: "image_id": "afd12e232ec04c7e77932076425916985d97c3486288fff2ba931630e84a9295", 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout: "repo_digests": [ 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout: "quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph@sha256:c4bf3497f7cb9cae2d0db734c6fa7ef173cd8dc715907bde16e4a68082d82e1d" 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2023-07-14T11:01:58.828 INFO:teuthology.orchestra.run.smithi045.stdout:} 2023-07-14T11:01:58.846 DEBUG:teuthology.orchestra.run.smithi040:> sudo mkdir -p /etc/ceph 2023-07-14T11:01:58.928 DEBUG:teuthology.orchestra.run.smithi045:> sudo mkdir -p /etc/ceph 2023-07-14T11:01:58.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2023-07-14T11:01:58.995 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 777 /etc/ceph 2023-07-14T11:01:59.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 777 /etc/ceph 2023-07-14T11:01:59.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2023-07-14T11:01:59.126 INFO:tasks.cephadm:Writing seed config... 2023-07-14T11:01:59.129 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2023-07-14T11:01:59.129 INFO:tasks.cephadm: override: [client] debug client = 20 2023-07-14T11:01:59.130 INFO:tasks.cephadm: override: [client] debug ms = 1 2023-07-14T11:01:59.130 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2023-07-14T11:01:59.130 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2023-07-14T11:01:59.131 INFO:tasks.cephadm: override: [global] ms die on skipped message = False 2023-07-14T11:01:59.131 INFO:tasks.cephadm: override: [mds] debug mds = 20 2023-07-14T11:01:59.132 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2023-07-14T11:01:59.132 INFO:tasks.cephadm: override: [mds] debug ms = 1 2023-07-14T11:01:59.132 INFO:tasks.cephadm: override: [mds] mds bal fragment size max = 10000 2023-07-14T11:01:59.133 INFO:tasks.cephadm: override: [mds] mds bal merge size = 5 2023-07-14T11:01:59.133 INFO:tasks.cephadm: override: [mds] mds bal split bits = 3 2023-07-14T11:01:59.134 INFO:tasks.cephadm: override: [mds] mds bal split size = 100 2023-07-14T11:01:59.134 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2023-07-14T11:01:59.134 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2023-07-14T11:01:59.135 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2023-07-14T11:01:59.135 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2023-07-14T11:01:59.135 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2023-07-14T11:01:59.136 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2023-07-14T11:01:59.136 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2023-07-14T11:01:59.137 INFO:tasks.cephadm: override: [mgr] cephsqlite lock renewal timeout = 900000 2023-07-14T11:01:59.137 INFO:tasks.cephadm: override: [mgr] debug cephsqlite = 20 2023-07-14T11:01:59.137 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-07-14T11:01:59.138 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-07-14T11:01:59.138 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-07-14T11:01:59.139 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-07-14T11:01:59.139 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-07-14T11:01:59.140 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2023-07-14T11:01:59.140 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2023-07-14T11:01:59.140 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2023-07-14T11:01:59.141 INFO:tasks.cephadm: override: [osd] debug bluefs = 20 2023-07-14T11:01:59.141 INFO:tasks.cephadm: override: [osd] debug bluestore = 20 2023-07-14T11:01:59.142 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-07-14T11:01:59.142 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-07-14T11:01:59.142 INFO:tasks.cephadm: override: [osd] debug rocksdb = 10 2023-07-14T11:01:59.143 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2023-07-14T11:01:59.143 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2023-07-14T11:01:59.143 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2023-07-14T11:01:59.144 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2023-07-14T11:01:59.144 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2023-07-14T11:01:59.145 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2023-07-14T11:01:59.145 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-07-14T11:01:59.145 INFO:tasks.cephadm: override: [osd] osd_max_omap_entries_per_request = 10000 2023-07-14T11:01:59.147 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:01:59.148 DEBUG:teuthology.orchestra.run.smithi040:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-07-14T11:01:59.192 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = bbe0b002-2235-11ee-9b32-001a4aab830c ms die on skipped message = False [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true bluestore block size = 96636764160 bluestore fsck on mount = True debug bluefs = 20 debug bluestore = 20 debug ms = 1 debug osd = 20 debug rocksdb = 10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 osd_max_omap_entries_per_request = 10000 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false cephsqlite lock renewal timeout = 900000 debug cephsqlite = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds bal fragment size max = 10000 mds bal merge size = 5 mds bal split bits = 3 mds bal split size = 100 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2023-07-14T11:01:59.193 DEBUG:teuthology.orchestra.run.smithi040:mon.a> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service 2023-07-14T11:01:59.237 DEBUG:teuthology.orchestra.run.smithi040:mgr.x> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x.service 2023-07-14T11:01:59.282 INFO:tasks.cephadm:Bootstrapping... 2023-07-14T11:01:59.282 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b -v bootstrap --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id x --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.40 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:01:59.305 INFO:journalctl@ceph.mon.a.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:01:59.351 INFO:journalctl@ceph.mgr.x.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:01:59.514 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-07-14T11:01:59.514 INFO:teuthology.orchestra.run.smithi040.stdout:cephadm ['--image', 'quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b', '-v', 'bootstrap', '--fsid', 'bbe0b002-2235-11ee-9b32-001a4aab830c', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'x', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.40', '--skip-admin-label'] 2023-07-14T11:01:59.687 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stdout 4.1.1 2023-07-14T11:01:59.687 INFO:teuthology.orchestra.run.smithi040.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-07-14T11:01:59.688 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying podman|docker is present... 2023-07-14T11:01:59.858 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stdout 4.1.1 2023-07-14T11:01:59.859 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying lvm2 is present... 2023-07-14T11:01:59.859 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying time synchronization is in place... 2023-07-14T11:01:59.874 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-07-14T11:01:59.874 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-07-14T11:01:59.887 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-07-14T11:01:59.887 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:01:59.900 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout enabled 2023-07-14T11:01:59.914 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout active 2023-07-14T11:01:59.914 INFO:teuthology.orchestra.run.smithi040.stdout:Unit chronyd.service is enabled and running 2023-07-14T11:01:59.914 INFO:teuthology.orchestra.run.smithi040.stdout:Repeating the final host check... 2023-07-14T11:02:00.084 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stdout 4.1.1 2023-07-14T11:02:00.084 INFO:teuthology.orchestra.run.smithi040.stdout:podman (/bin/podman) version 4.1.1 is present 2023-07-14T11:02:00.084 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl is present 2023-07-14T11:02:00.084 INFO:teuthology.orchestra.run.smithi040.stdout:lvcreate is present 2023-07-14T11:02:00.098 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-07-14T11:02:00.098 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-07-14T11:02:00.110 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-07-14T11:02:00.111 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:02:00.124 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout enabled 2023-07-14T11:02:00.137 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout active 2023-07-14T11:02:00.137 INFO:teuthology.orchestra.run.smithi040.stdout:Unit chronyd.service is enabled and running 2023-07-14T11:02:00.138 INFO:teuthology.orchestra.run.smithi040.stdout:Host looks OK 2023-07-14T11:02:00.138 INFO:teuthology.orchestra.run.smithi040.stdout:Cluster fsid: bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:00.139 INFO:teuthology.orchestra.run.smithi040.stdout:Acquiring lock 139728911182312 on /run/cephadm/bbe0b002-2235-11ee-9b32-001a4aab830c.lock 2023-07-14T11:02:00.139 INFO:teuthology.orchestra.run.smithi040.stdout:Lock 139728911182312 acquired on /run/cephadm/bbe0b002-2235-11ee-9b32-001a4aab830c.lock 2023-07-14T11:02:00.139 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying IP 172.21.15.40 port 3300 ... 2023-07-14T11:02:00.140 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying IP 172.21.15.40 port 6789 ... 2023-07-14T11:02:00.140 INFO:teuthology.orchestra.run.smithi040.stdout:Base mon IP(s) is [172.21.15.40:3300, 172.21.15.40:6789], mon addrv is [v2:172.21.15.40:3300,v1:172.21.15.40:6789] 2023-07-14T11:02:00.147 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-07-14T11:02:00.147 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.40 metric 100 2023-07-14T11:02:00.154 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-07-14T11:02:00.154 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-07-14T11:02:00.154 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1750sec hoplimit 64 pref medium 2023-07-14T11:02:00.160 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-07-14T11:02:00.161 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-07-14T11:02:00.161 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-07-14T11:02:00.161 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-07-14T11:02:00.161 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:12cc/64 scope link 2023-07-14T11:02:00.161 INFO:teuthology.orchestra.run.smithi040.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-07-14T11:02:00.162 INFO:teuthology.orchestra.run.smithi040.stdout:Mon IP `172.21.15.40` is in CIDR network `172.21.0.0/20` 2023-07-14T11:02:00.162 INFO:teuthology.orchestra.run.smithi040.stdout:Mon IP `172.21.15.40` is in CIDR network `172.21.0.0/20` 2023-07-14T11:02:00.162 INFO:teuthology.orchestra.run.smithi040.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-07-14T11:02:00.162 INFO:teuthology.orchestra.run.smithi040.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-07-14T11:02:00.163 INFO:teuthology.orchestra.run.smithi040.stdout:Pulling container image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b... 2023-07-14T11:02:00.637 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stdout afd12e232ec04c7e77932076425916985d97c3486288fff2ba931630e84a9295 2023-07-14T11:02:00.637 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Trying to pull quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b... 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Getting image source signatures 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Copying blob sha256:d8eac6e0ff3c58bd5607bffacb1c2214d20cdd3ff8cfaf23cfeb0cc60b2d0d6b 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Copying blob sha256:6022c611d474f2b306ddd1edfc8360123ece5266119096f25fde843d7967e636 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Copying config sha256:afd12e232ec04c7e77932076425916985d97c3486288fff2ba931630e84a9295 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Writing manifest to image destination 2023-07-14T11:02:00.638 INFO:teuthology.orchestra.run.smithi040.stdout:/bin/podman: stderr Storing signatures 2023-07-14T11:02:01.613 INFO:teuthology.orchestra.run.smithi040.stdout:ceph: stdout ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev) 2023-07-14T11:02:01.613 INFO:teuthology.orchestra.run.smithi040.stdout:Ceph version: ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev) 2023-07-14T11:02:01.613 INFO:teuthology.orchestra.run.smithi040.stdout:Extracting ceph user uid/gid from container image... 2023-07-14T11:02:02.495 INFO:teuthology.orchestra.run.smithi040.stdout:stat: stdout 167 167 2023-07-14T11:02:02.495 INFO:teuthology.orchestra.run.smithi040.stdout:Creating initial keys... 2023-07-14T11:02:03.370 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph-authtool: stdout AQAqK7FkglMQNRAAghvrKj+el0V3Oa5NTE9wEw== 2023-07-14T11:02:04.242 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph-authtool: stdout AQArK7Fkk88KLhAAtt2JysYMu5E8YD3Zoqe59g== 2023-07-14T11:02:05.116 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph-authtool: stdout AQAsK7FkFIqjJRAApMNL4OwQyJkO9oRtF4jFog== 2023-07-14T11:02:05.117 INFO:teuthology.orchestra.run.smithi040.stdout:Creating initial monmap... 2023-07-14T11:02:05.997 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-07-14T11:02:05.997 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2023-07-14T11:02:05.997 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:05.998 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-07-14T11:02:05.999 INFO:teuthology.orchestra.run.smithi040.stdout:monmaptool for a [v2:172.21.15.40:3300,v1:172.21.15.40:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-07-14T11:02:06.000 INFO:teuthology.orchestra.run.smithi040.stdout:setting min_mon_release = pacific 2023-07-14T11:02:06.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: set fsid to bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:06.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-07-14T11:02:06.001 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:02:06.001 INFO:teuthology.orchestra.run.smithi040.stdout:Creating mon... 2023-07-14T11:02:06.874 INFO:teuthology.orchestra.run.smithi040.stdout:create mon.a on 2023-07-14T11:02:07.163 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-07-14T11:02:07.372 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-07-14T11:02:07.617 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target → /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target. 2023-07-14T11:02:07.617 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target → /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target. 2023-07-14T11:02:07.998 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a 2023-07-14T11:02:07.998 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service: Unit ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service not loaded. 2023-07-14T11:02:08.208 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target.wants/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service → /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@.service. 2023-07-14T11:02:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:08 smithi040 systemd[1]: Starting Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:02:08.962 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:08 smithi040 podman[97414]: 2023-07-14T11:02:09.019 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-07-14T11:02:09.019 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout disabled 2023-07-14T11:02:09.035 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-07-14T11:02:09.035 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:02:09.036 INFO:teuthology.orchestra.run.smithi040.stdout:firewalld.service is not enabled 2023-07-14T11:02:09.036 INFO:teuthology.orchestra.run.smithi040.stdout:Not possible to enable service . firewalld.service is not available 2023-07-14T11:02:09.037 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mon to start... 2023-07-14T11:02:09.037 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mon... 2023-07-14T11:02:09.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:08 smithi040 bash[97414]: 44b10dd98bd3075f2799d9ba0287273341f685d22a204ec9803694e3c65df5a2 2023-07-14T11:02:09.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:08 smithi040 systemd[1]: Started Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:02:09.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:09 smithi040 ceph-mon[97454]: mkfs bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:09.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:09 smithi040 ceph-mon[97454]: mon.a is new leader, mons a in quorum (ranks 0) 2023-07-14T11:02:10.120 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout cluster: 2023-07-14T11:02:10.120 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout id: bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:10.120 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout services: 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.678823s) 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout data: 2023-07-14T11:02:10.121 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout pgs: 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.588+0000 7fb7caf31700 1 Processor -- start 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.588+0000 7fb7caf31700 1 -- start start 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.589+0000 7fb7caf31700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:10.122 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.589+0000 7fb7caf31700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7c407a880 con 0x7fb7c407bdf0 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.590+0000 7fb7c8ccd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.590+0000 7fb7c8ccd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54194/0 (socket says 172.21.15.40:54194) 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.590+0000 7fb7c8ccd700 1 -- 172.21.15.40:0/3812082373 learned_addr learned my addr 172.21.15.40:0/3812082373 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.590+0000 7fb7c8ccd700 1 -- 172.21.15.40:0/3812082373 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7c407a9c0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7c8ccd700 1 --2- 172.21.15.40:0/3812082373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb7b80098a0 tx=0x7fb7b8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1aa0c18675802158 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7c37fe700 1 -- 172.21.15.40:0/3812082373 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7b8004160 con 0x7fb7c407bdf0 2023-07-14T11:02:10.123 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7c37fe700 1 -- 172.21.15.40:0/3812082373 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb7b80042c0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7c37fe700 1 -- 172.21.15.40:0/3812082373 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7b80045a0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7caf31700 1 -- 172.21.15.40:0/3812082373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 msgr2=0x7fb7c407a340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.591+0000 7fb7caf31700 1 --2- 172.21.15.40:0/3812082373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fb7b80098a0 tx=0x7fb7b8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.592+0000 7fb7caf31700 1 -- 172.21.15.40:0/3812082373 shutdown_connections 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.592+0000 7fb7caf31700 1 --2- 172.21.15.40:0/3812082373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c407a340 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.592+0000 7fb7caf31700 1 -- 172.21.15.40:0/3812082373 >> 172.21.15.40:0/3812082373 conn(0x7fb7c4100910 msgr2=0x7fb7c4102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.592+0000 7fb7caf31700 1 -- 172.21.15.40:0/3812082373 shutdown_connections 2023-07-14T11:02:10.124 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.592+0000 7fb7caf31700 1 -- 172.21.15.40:0/3812082373 wait complete. 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.593+0000 7fb7caf31700 1 Processor -- start 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.593+0000 7fb7caf31700 1 -- start start 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.593+0000 7fb7caf31700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.593+0000 7fb7caf31700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7c419aea0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c8ccd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c8ccd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54202/0 (socket says 172.21.15.40:54202) 2023-07-14T11:02:10.125 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c8ccd700 1 -- 172.21.15.40:0/2900374251 learned_addr learned my addr 172.21.15.40:0/2900374251 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c8ccd700 1 -- 172.21.15.40:0/2900374251 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7b8009550 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c8ccd700 1 --2- 172.21.15.40:0/2900374251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fb7b8004000 tx=0x7fb7b8004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.594+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7b8011630 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fb7b8011790 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7c419b0a0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7b8011a30 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7c419b4c0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.126 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 660+0+0 (secure 0 0 0) 0x7fb7b8022070 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.595+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fb7b801c450 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.596+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb7c4194790 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.599+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72459 (secure 0 0 0) 0x7fb7b801b450 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.674+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status"} v 0) v1 -- 0x7fb7c402c850 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.675+0000 7fb7c1ffb700 1 -- 172.21.15.40:0/2900374251 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) v1 ==== 54+0+317 (secure 0 0 0) 0x7fb7b801b5f0 con 0x7fb7c407bdf0 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.675+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 msgr2=0x7fb7c419a960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:10.127 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.675+0000 7fb7caf31700 1 --2- 172.21.15.40:0/2900374251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fb7b8004000 tx=0x7fb7b8004750 comp rx=0 tx=0).stop 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.675+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 shutdown_connections 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.676+0000 7fb7caf31700 1 --2- 172.21.15.40:0/2900374251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7c407bdf0 0x7fb7c419a960 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.676+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 >> 172.21.15.40:0/2900374251 conn(0x7fb7c4100910 msgr2=0x7fb7c4102190 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.676+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 shutdown_connections 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:09.676+0000 7fb7caf31700 1 -- 172.21.15.40:0/2900374251 wait complete. 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:mon is available 2023-07-14T11:02:10.128 INFO:teuthology.orchestra.run.smithi040.stdout:Assimilating anything we can from ceph.conf... 2023-07-14T11:02:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: mon.a is new leader, mons a in quorum (ranks 0) 2023-07-14T11:02:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: monmap e1: 1 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0]} removed_ranks: {} 2023-07-14T11:02:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: fsmap 2023-07-14T11:02:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: osdmap e1: 0 total, 0 up, 0 in 2023-07-14T11:02:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: mgrmap e1: no daemons active 2023-07-14T11:02:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:10 smithi040 ceph-mon[97454]: from='client.? 172.21.15.40:0/2900374251' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-07-14T11:02:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:11 smithi040 ceph-mon[97454]: from='client.? 172.21.15.40:0/4034233149' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-07-14T11:02:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:11 smithi040 ceph-mon[97454]: from='client.? 172.21.15.40:0/4034233149' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-07-14T11:02:11.202 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:11.202 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [global] 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout fsid = bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.40:3300,v1:172.21.15.40:6789] 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-07-14T11:02:11.203 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [mgr] 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [osd] 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2023-07-14T11:02:11.204 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.662+0000 7efe90b6e700 1 Processor -- start 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.662+0000 7efe90b6e700 1 -- start start 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.662+0000 7efe90b6e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.662+0000 7efe90b6e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe8c07c1c0 con 0x7efe8c07bdb0 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.663+0000 7efe8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:11.205 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.663+0000 7efe8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54210/0 (socket says 172.21.15.40:54210) 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.663+0000 7efe8a59c700 1 -- 172.21.15.40:0/305621314 learned_addr learned my addr 172.21.15.40:0/305621314 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.664+0000 7efe8a59c700 1 -- 172.21.15.40:0/305621314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe8c07a840 con 0x7efe8c07bdb0 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.664+0000 7efe8a59c700 1 --2- 172.21.15.40:0/305621314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7efe7c0098a0 tx=0x7efe7c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e54bc90d8fcdc3bd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.664+0000 7efe8959a700 1 -- 172.21.15.40:0/305621314 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe7c004160 con 0x7efe8c07bdb0 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.664+0000 7efe8959a700 1 -- 172.21.15.40:0/305621314 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efe7c0042c0 con 0x7efe8c07bdb0 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.664+0000 7efe8959a700 1 -- 172.21.15.40:0/305621314 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe7c0045a0 con 0x7efe8c07bdb0 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 -- 172.21.15.40:0/305621314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 msgr2=0x7efe8c07a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:11.206 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 --2- 172.21.15.40:0/305621314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7efe7c0098a0 tx=0x7efe7c009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 -- 172.21.15.40:0/305621314 shutdown_connections 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 --2- 172.21.15.40:0/305621314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c07a300 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 -- 172.21.15.40:0/305621314 >> 172.21.15.40:0/305621314 conn(0x7efe8c100910 msgr2=0x7efe8c102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 -- 172.21.15.40:0/305621314 shutdown_connections 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.665+0000 7efe90b6e700 1 -- 172.21.15.40:0/305621314 wait complete. 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe90b6e700 1 Processor -- start 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe90b6e700 1 -- start start 2023-07-14T11:02:11.207 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe90b6e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe90b6e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efe8c19ae70 con 0x7efe8c07bdb0 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54218/0 (socket says 172.21.15.40:54218) 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.666+0000 7efe8a59c700 1 -- 172.21.15.40:0/4034233149 learned_addr learned my addr 172.21.15.40:0/4034233149 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe8a59c700 1 -- 172.21.15.40:0/4034233149 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efe7c009550 con 0x7efe8c07bdb0 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe8a59c700 1 --2- 172.21.15.40:0/4034233149 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7efe7c004000 tx=0x7efe7c004750 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:11.208 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe7c011630 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7efe7c011790 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7efe7c011a30 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe90b6e700 1 -- 172.21.15.40:0/4034233149 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efe8c19b070 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.667+0000 7efe90b6e700 1 -- 172.21.15.40:0/4034233149 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efe8c19b490 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.668+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 660+0+0 (secure 0 0 0) 0x7efe7c022070 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.668+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7efe7c01c790 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.669+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efe700052f0 con 0x7efe8c07bdb0 2023-07-14T11:02:11.209 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.671+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72459 (secure 0 0 0) 0x7efe7c01bcf0 con 0x7efe8c07bdb0 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.745+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7efe70002400 con 0x7efe8c07bdb0 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.749+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 7 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efe7c01be90 con 0x7efe8c07bdb0 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.749+0000 7efe7b7fe700 1 -- 172.21.15.40:0/4034233149 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) v1 ==== 70+0+429 (secure 0 0 0) 0x7efe7c037610 con 0x7efe8c07bdb0 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.750+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 msgr2=0x7efe8c19a930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.750+0000 7efe797fa700 1 --2- 172.21.15.40:0/4034233149 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7efe7c004000 tx=0x7efe7c004750 comp rx=0 tx=0).stop 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.751+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 shutdown_connections 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.751+0000 7efe797fa700 1 --2- 172.21.15.40:0/4034233149 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efe8c07bdb0 0x7efe8c19a930 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:11.210 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.751+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 >> 172.21.15.40:0/4034233149 conn(0x7efe8c100910 msgr2=0x7efe8c102200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:11.211 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.751+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 shutdown_connections 2023-07-14T11:02:11.211 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:10.751+0000 7efe797fa700 1 -- 172.21.15.40:0/4034233149 wait complete. 2023-07-14T11:02:11.211 INFO:teuthology.orchestra.run.smithi040.stdout:Generating new minimal ceph.conf... 2023-07-14T11:02:12.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.728+0000 7f7c6cab9700 1 Processor -- start 2023-07-14T11:02:12.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.728+0000 7f7c6cab9700 1 -- start start 2023-07-14T11:02:12.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.728+0000 7f7c6cab9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:12.283 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.728+0000 7f7c6cab9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c6807c1c0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.283 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:12.283 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54230/0 (socket says 172.21.15.40:54230) 2023-07-14T11:02:12.283 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6659c700 1 -- 172.21.15.40:0/3329023804 learned_addr learned my addr 172.21.15.40:0/3329023804 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:12.284 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6659c700 1 -- 172.21.15.40:0/3329023804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c6807a840 con 0x7f7c6807bdb0 2023-07-14T11:02:12.284 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6659c700 1 --2- 172.21.15.40:0/3329023804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f7c580098a0 tx=0x7f7c58009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=231df0fbf8f9442a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:12.284 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6559a700 1 -- 172.21.15.40:0/3329023804 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c58004160 con 0x7f7c6807bdb0 2023-07-14T11:02:12.285 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.729+0000 7f7c6559a700 1 -- 172.21.15.40:0/3329023804 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7c580042c0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.285 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6559a700 1 -- 172.21.15.40:0/3329023804 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c58011a00 con 0x7f7c6807bdb0 2023-07-14T11:02:12.285 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 -- 172.21.15.40:0/3329023804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 msgr2=0x7f7c6807a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:12.286 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 --2- 172.21.15.40:0/3329023804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f7c580098a0 tx=0x7f7c58009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:12.286 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 -- 172.21.15.40:0/3329023804 shutdown_connections 2023-07-14T11:02:12.286 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 --2- 172.21.15.40:0/3329023804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6807a300 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:12.286 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 -- 172.21.15.40:0/3329023804 >> 172.21.15.40:0/3329023804 conn(0x7f7c68100910 msgr2=0x7f7c68102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:12.287 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 -- 172.21.15.40:0/3329023804 shutdown_connections 2023-07-14T11:02:12.287 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.730+0000 7f7c6cab9700 1 -- 172.21.15.40:0/3329023804 wait complete. 2023-07-14T11:02:12.287 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.731+0000 7f7c6cab9700 1 Processor -- start 2023-07-14T11:02:12.287 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.731+0000 7f7c6cab9700 1 -- start start 2023-07-14T11:02:12.288 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6cab9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:12.288 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6cab9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c6819b6a0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.288 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:12.288 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54242/0 (socket says 172.21.15.40:54242) 2023-07-14T11:02:12.289 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6659c700 1 -- 172.21.15.40:0/4233791003 learned_addr learned my addr 172.21.15.40:0/4233791003 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:12.289 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.732+0000 7f7c6659c700 1 -- 172.21.15.40:0/4233791003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c58009550 con 0x7f7c6807bdb0 2023-07-14T11:02:12.289 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c6659c700 1 --2- 172.21.15.40:0/4233791003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f7c58006af0 tx=0x7f7c58011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:12.289 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c58003a40 con 0x7f7c6807bdb0 2023-07-14T11:02:12.290 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7c58003ba0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.290 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7c5801a700 con 0x7f7c6807bdb0 2023-07-14T11:02:12.290 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c6819b8a0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.291 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c6819bcc0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.291 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 660+0+0 (secure 0 0 0) 0x7f7c5801ac30 con 0x7f7c6807bdb0 2023-07-14T11:02:12.291 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.733+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f7c580193c0 con 0x7f7c6807bdb0 2023-07-14T11:02:12.291 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.734+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7c6810b580 con 0x7f7c6807bdb0 2023-07-14T11:02:12.292 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.737+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72459 (secure 0 0 0) 0x7f7c58024770 con 0x7f7c6807bdb0 2023-07-14T11:02:12.292 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.811+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f7c6819ed50 con 0x7f7c6807bdb0 2023-07-14T11:02:12.292 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.811+0000 7f7c577fe700 1 -- 172.21.15.40:0/4233791003 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) v1 ==== 76+0+175 (secure 0 0 0) 0x7f7c58024910 con 0x7f7c6807bdb0 2023-07-14T11:02:12.293 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 msgr2=0x7f7c6819b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:12.293 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 --2- 172.21.15.40:0/4233791003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f7c58006af0 tx=0x7f7c58011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:12.293 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 shutdown_connections 2023-07-14T11:02:12.293 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 --2- 172.21.15.40:0/4233791003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c6807bdb0 0x7f7c6819b160 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:12.294 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 >> 172.21.15.40:0/4233791003 conn(0x7f7c68100910 msgr2=0x7f7c68101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:12.294 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 shutdown_connections 2023-07-14T11:02:12.294 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:11.812+0000 7f7c6cab9700 1 -- 172.21.15.40:0/4233791003 wait complete. 2023-07-14T11:02:12.294 INFO:teuthology.orchestra.run.smithi040.stdout:Restarting the monitor... 2023-07-14T11:02:12.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:12 smithi040 ceph-mon[97454]: from='client.? 172.21.15.40:0/4233791003' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:02:12.742 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:12 smithi040 systemd[1]: Stopping Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:02:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a[97430]: 2023-07-14T11:02:12.737+0000 7f13317cd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:02:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a[97430]: 2023-07-14T11:02:12.737+0000 7f13317cd700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-07-14T11:02:13.475 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:13 smithi040 bash[97788]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a 2023-07-14T11:02:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:13 smithi040 systemd[1]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service: Succeeded. 2023-07-14T11:02:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:13 smithi040 systemd[1]: Stopped Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:02:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:13 smithi040 systemd[1]: Starting Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:02:14.204 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:13 smithi040 podman[97914]: 2023-07-14T11:02:14.205 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 bash[97914]: 65654455f8c8267188254b32b470309faa208f117289a932648a067749e6baab 2023-07-14T11:02:14.205 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: set uid:gid to 167:167 (ceph:ceph) 2023-07-14T11:02:14.219 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev), process ceph-mon, pid 7 2023-07-14T11:02:14.219 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: pidfile_write: ignore empty --pid-file 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: load: jerasure load: lrc 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: RocksDB version: 7.9.2 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Git sha 0 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Compile date 2023-07-13 18:29:45 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: DB SUMMARY 2023-07-14T11:02:14.220 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: DB Session ID: 32DXK3VGJOEYXW8IRVC2 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: CURRENT file: CURRENT 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: IDENTITY file: IDENTITY 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 98172 ; 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.error_if_exists: 0 2023-07-14T11:02:14.221 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.create_if_missing: 0 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.paranoid_checks: 1 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.flush_verify_memtable_count: 1 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.env: 0x5638c322f720 2023-07-14T11:02:14.222 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.fs: PosixFileSystem 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.info_log: 0x5638c3e45280 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_file_opening_threads: 16 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.statistics: (nil) 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.use_fsync: 0 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_log_file_size: 0 2023-07-14T11:02:14.223 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.log_file_time_to_roll: 0 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.keep_log_file_num: 1000 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.recycle_log_file_num: 0 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_fallocate: 1 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_mmap_reads: 0 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_mmap_writes: 0 2023-07-14T11:02:14.224 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.use_direct_reads: 0 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.create_missing_column_families: 0 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.db_log_dir: 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.wal_dir: 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.table_cache_numshardbits: 6 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.WAL_ttl_seconds: 0 2023-07-14T11:02:14.225 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.WAL_size_limit_MB: 0 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.is_fd_close_on_exec: 1 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.advise_random_on_open: 1 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.db_write_buffer_size: 0 2023-07-14T11:02:14.226 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.write_buffer_manager: 0x5638c3d663c0 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.use_adaptive_mutex: 0 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.rate_limiter: (nil) 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.wal_recovery_mode: 2 2023-07-14T11:02:14.227 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enable_thread_tracking: 0 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enable_pipelined_write: 0 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.unordered_write: 0 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-07-14T11:02:14.228 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.row_cache: None 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.wal_filter: None 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_ingest_behind: 0 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.two_write_queues: 0 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.manual_wal_flush: 0 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.wal_compression: 0 2023-07-14T11:02:14.229 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.atomic_flush: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.persist_stats_to_disk: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.write_dbid_to_manifest: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.log_readahead_size: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.best_efforts_recovery: 0 2023-07-14T11:02:14.230 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.allow_data_in_errors: 0 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.db_host_id: __hostname__ 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enforce_single_del_contracts: true 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_background_jobs: 2 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_background_compactions: -1 2023-07-14T11:02:14.231 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_subcompactions: 1 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.delayed_write_rate : 16777216 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_total_wal_size: 0 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.stats_dump_period_sec: 600 2023-07-14T11:02:14.232 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.stats_persist_period_sec: 600 2023-07-14T11:02:14.233 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-07-14T11:02:14.233 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_open_files: -1 2023-07-14T11:02:14.261 INFO:teuthology.orchestra.run.smithi040.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2023-07-14T11:02:14.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bytes_per_sync: 0 2023-07-14T11:02:14.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.wal_bytes_per_sync: 0 2023-07-14T11:02:14.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.strict_bytes_per_sync: 0 2023-07-14T11:02:14.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_readahead_size: 0 2023-07-14T11:02:14.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_background_flushes: -1 2023-07-14T11:02:14.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Compression algorithms supported: 2023-07-14T11:02:14.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-07-14T11:02:14.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kZSTD supported: 0 2023-07-14T11:02:14.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kXpressCompression supported: 0 2023-07-14T11:02:14.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kLZ4HCCompression supported: 1 2023-07-14T11:02:14.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kZlibCompression supported: 1 2023-07-14T11:02:14.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kSnappyCompression supported: 1 2023-07-14T11:02:14.552 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kLZ4Compression supported: 1 2023-07-14T11:02:14.552 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: kBZip2Compression supported: 0 2023-07-14T11:02:14.552 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Fast CRC32 supported: Supported on x86 2023-07-14T11:02:14.552 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: DMutex implementation: pthread_mutex_t 2023-07-14T11:02:14.553 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2023-07-14T11:02:14.553 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-07-14T11:02:14.553 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-07-14T11:02:14.553 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.merge_operator: 2023-07-14T11:02:14.553 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_filter: None 2023-07-14T11:02:14.554 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_filter_factory: None 2023-07-14T11:02:14.554 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.sst_partitioner_factory: None 2023-07-14T11:02:14.554 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.memtable_factory: SkipListFactory 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.table_factory: BlockBasedTable 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5638c3e45380) 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout: cache_index_and_filter_blocks: 1 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-07-14T11:02:14.555 INFO:journalctl@ceph.mon.a.smithi040.stdout: pin_top_level_index_and_filter: 1 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: index_type: 0 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: data_block_index_type: 0 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: index_shortening: 1 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: data_block_hash_table_util_ratio: 0.750000 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: checksum: 4 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: no_block_cache: 0 2023-07-14T11:02:14.556 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_cache: 0x5638c3e051f0 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_cache_name: BinnedLRUCache 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_cache_options: 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: capacity : 536870912 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: num_shard_bits : 4 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: strict_capacity_limit : 0 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: high_pri_pool_ratio: 0.000 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_cache_compressed: (nil) 2023-07-14T11:02:14.557 INFO:journalctl@ceph.mon.a.smithi040.stdout: persistent_cache: (nil) 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_size: 4096 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_size_deviation: 10 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_restart_interval: 16 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: index_block_restart_interval: 1 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: metadata_block_size: 4096 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: partition_filters: 0 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: use_delta_encoding: 1 2023-07-14T11:02:14.558 INFO:journalctl@ceph.mon.a.smithi040.stdout: filter_policy: bloomfilter 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: whole_key_filtering: 1 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: verify_compression: 0 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: read_amp_bytes_per_bit: 0 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: format_version: 5 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: enable_index_compression: 1 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: block_align: 0 2023-07-14T11:02:14.559 INFO:journalctl@ceph.mon.a.smithi040.stdout: max_auto_readahead_size: 262144 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout: prepopulate_block_cache: 0 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout: initial_auto_readahead_size: 8192 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout: num_file_reads_for_auto_readahead: 2 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.write_buffer_size: 33554432 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_write_buffer_number: 2 2023-07-14T11:02:14.560 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression: NoCompression 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression: Disabled 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.prefix_extractor: nullptr 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.num_levels: 7 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-07-14T11:02:14.561 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-07-14T11:02:14.562 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.window_bits: -14 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.level: 32767 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.strategy: 0 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:02:14.563 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.enabled: false 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.target_file_size_base: 67108864 2023-07-14T11:02:14.564 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.target_file_size_multiplier: 1 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-07-14T11:02:14.565 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.arena_block_size: 1048576 2023-07-14T11:02:14.566 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.disable_auto_compactions: 0 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-07-14T11:02:14.567 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.table_properties_collectors: 2023-07-14T11:02:14.568 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.inplace_update_support: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.inplace_update_num_locks: 10000 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.memtable_huge_page_size: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.bloom_locality: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.max_successive_merges: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.optimize_filters_for_hits: 0 2023-07-14T11:02:14.569 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.paranoid_file_checks: 0 2023-07-14T11:02:14.570 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.force_consistency_checks: 1 2023-07-14T11:02:14.570 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.report_bg_io_stats: 0 2023-07-14T11:02:14.570 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.ttl: 2592000 2023-07-14T11:02:14.570 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.periodic_compaction_seconds: 0 2023-07-14T11:02:14.570 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enable_blob_files: false 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.min_blob_size: 0 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_file_size: 268435456 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_compression_type: NoCompression 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.enable_blob_garbage_collection: false 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-07-14T11:02:14.571 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.blob_file_starting_level: 0 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2023-07-14T11:02:14.572 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0b3e8f63-8a64-4805-930e-252443acc3f8 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332534211172, "job": 1, "event": "recovery_started", "wal_files": [9]} 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332534213007, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 93482, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 312, "table_properties": {"data_size": 91482, "index_size": 282, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 13509, "raw_average_key_size": 49, "raw_value_size": 84111, "raw_average_value_size": 310, "num_data_blocks": 11, "num_entries": 271, "num_filter_entries": 271, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1689332534, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0b3e8f63-8a64-4805-930e-252443acc3f8", "db_session_id": "32DXK3VGJOEYXW8IRVC2", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332534213208, "job": 1, "event": "recovery_finished"} 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-07-14T11:02:14.573 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5638c3e9e000 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: DB pointer 0x5638c3e8a000 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout: ** DB Stats ** 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-07-14T11:02:14.574 INFO:journalctl@ceph.mon.a.smithi040.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: ** Compaction Stats [default] ** 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:02:14.575 INFO:journalctl@ceph.mon.a.smithi040.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: L0 2/0 93.13 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: Sum 2/0 93.13 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: ** Compaction Stats [default] ** 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-07-14T11:02:14.576 INFO:journalctl@ceph.mon.a.smithi040.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 75.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-07-14T11:02:14.577 INFO:journalctl@ceph.mon.a.smithi040.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: AddFile(Keys): cumulative 0, interval 0 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: Cumulative compaction: 0.00 GB write, 13.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: Interval compaction: 0.00 GB write, 13.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: Block cache BinnedLRUCache@0x5638c3e051f0#7 capacity: 512.00 MB usage: 1.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.89 KB,0.000169873%) IndexBlock(2,0.47 KB,8.9407e-05%) Misc(1,0.00 KB,0%) 2023-07-14T11:02:14.578 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout: ** File Read Latency Histogram By Level [default] ** 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: starting mon.a rank 0 at public addrs [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] at bind addrs [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???) e1 preinit fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).mds e1 new map 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).mds e1 print_map 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout: e1 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-07-14T11:02:14.579 INFO:journalctl@ceph.mon.a.smithi040.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout: legacy client fscid: -1 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout: 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout: No filesystems configured 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:02:14.580 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 systemd[1]: Started Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mon.a is new leader, mons a in quorum (ranks 0) 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: monmap e1: 1 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0]} removed_ranks: {} 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: fsmap 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: osdmap e1: 0 total, 0 up, 0 in 2023-07-14T11:02:14.581 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:14 smithi040 ceph-mon[97954]: mgrmap e1: no daemons active 2023-07-14T11:02:15.344 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.834+0000 7fddecd19700 1 Processor -- start 2023-07-14T11:02:15.345 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.834+0000 7fddecd19700 1 -- start start 2023-07-14T11:02:15.345 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.835+0000 7fddecd19700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:15.345 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.835+0000 7fddecd19700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdde807c1c0 con 0x7fdde807bdb0 2023-07-14T11:02:15.345 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.835+0000 7fdde659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:15.345 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.835+0000 7fdde659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58842/0 (socket says 172.21.15.40:58842) 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.835+0000 7fdde659c700 1 -- 172.21.15.40:0/949685614 learned_addr learned my addr 172.21.15.40:0/949685614 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.836+0000 7fdde659c700 1 -- 172.21.15.40:0/949685614 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdde807a840 con 0x7fdde807bdb0 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.836+0000 7fdde659c700 1 --2- 172.21.15.40:0/949685614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fddd80098a0 tx=0x7fddd8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8b261bc6393cb382 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.836+0000 7fdde559a700 1 -- 172.21.15.40:0/949685614 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fddd8004160 con 0x7fdde807bdb0 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.836+0000 7fdde559a700 1 -- 172.21.15.40:0/949685614 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fddd80042c0 con 0x7fdde807bdb0 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.836+0000 7fdde559a700 1 -- 172.21.15.40:0/949685614 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fddd8011a00 con 0x7fdde807bdb0 2023-07-14T11:02:15.346 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 -- 172.21.15.40:0/949685614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 msgr2=0x7fdde807a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 --2- 172.21.15.40:0/949685614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7fddd80098a0 tx=0x7fddd8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 -- 172.21.15.40:0/949685614 shutdown_connections 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 --2- 172.21.15.40:0/949685614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde807a300 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 -- 172.21.15.40:0/949685614 >> 172.21.15.40:0/949685614 conn(0x7fdde8100910 msgr2=0x7fdde8102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 -- 172.21.15.40:0/949685614 shutdown_connections 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.837+0000 7fddecd19700 1 -- 172.21.15.40:0/949685614 wait complete. 2023-07-14T11:02:15.347 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.838+0000 7fddecd19700 1 Processor -- start 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.838+0000 7fddecd19700 1 -- start start 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fddecd19700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fddecd19700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdde819b6a0 con 0x7fdde807bdb0 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fdde659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fdde659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58850/0 (socket says 172.21.15.40:58850) 2023-07-14T11:02:15.348 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fdde659c700 1 -- 172.21.15.40:0/2327103462 learned_addr learned my addr 172.21.15.40:0/2327103462 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fdde659c700 1 -- 172.21.15.40:0/2327103462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fddd8009550 con 0x7fdde807bdb0 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.839+0000 7fdde659c700 1 --2- 172.21.15.40:0/2327103462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fddd8006af0 tx=0x7fddd8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fddd8003a40 con 0x7fdde807bdb0 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fddd8003ba0 con 0x7fdde807bdb0 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdde819b8a0 con 0x7fdde807bdb0 2023-07-14T11:02:15.349 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fddd801a700 con 0x7fdde807bdb0 2023-07-14T11:02:15.351 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdde819bcc0 con 0x7fdde807bdb0 2023-07-14T11:02:15.351 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 660+0+0 (secure 0 0 0) 0x7fddd801ac30 con 0x7fdde807bdb0 2023-07-14T11:02:15.351 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.840+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fddd8019700 con 0x7fdde807bdb0 2023-07-14T11:02:15.351 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.841+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdde8061b40 con 0x7fdde807bdb0 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.844+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72459 (secure 0 0 0) 0x7fddd802de40 con 0x7fdde807bdb0 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.917+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) v1 -- 0x7fdde819e700 con 0x7fdde807bdb0 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.920+0000 7fddd77fe700 1 -- 172.21.15.40:0/2327103462 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) v1 ==== 124+0+0 (secure 0 0 0) 0x7fddd8024ad0 con 0x7fdde807bdb0 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 msgr2=0x7fdde819b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 --2- 172.21.15.40:0/2327103462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7fddd8006af0 tx=0x7fddd8011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 shutdown_connections 2023-07-14T11:02:15.352 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 --2- 172.21.15.40:0/2327103462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdde807bdb0 0x7fdde819b160 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:15.353 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 >> 172.21.15.40:0/2327103462 conn(0x7fdde8100910 msgr2=0x7fdde8101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:15.353 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 shutdown_connections 2023-07-14T11:02:15.353 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:14.921+0000 7fddecd19700 1 -- 172.21.15.40:0/2327103462 wait complete. 2023-07-14T11:02:15.353 INFO:teuthology.orchestra.run.smithi040.stdout:Wrote config to /etc/ceph/ceph.conf 2023-07-14T11:02:15.364 INFO:teuthology.orchestra.run.smithi040.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:02:15.365 INFO:teuthology.orchestra.run.smithi040.stdout:Creating mgr... 2023-07-14T11:02:15.365 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying port 9283 ... 2023-07-14T11:02:15.366 INFO:teuthology.orchestra.run.smithi040.stdout:Verifying port 8765 ... 2023-07-14T11:02:15.738 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x 2023-07-14T11:02:15.739 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x.service: Unit ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x.service not loaded. 2023-07-14T11:02:15.952 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c.target.wants/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x.service → /etc/systemd/system/ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@.service. 2023-07-14T11:02:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:15 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2327103462' entity='client.admin' 2023-07-14T11:02:16.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:15 smithi040 systemd[1]: Starting Ceph mgr.x for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:02:16.740 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:16 smithi040 podman[98186]: 2023-07-14T11:02:16.740 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:16 smithi040 bash[98186]: 3b69bcb589de290e6b85218abba8a4b5fb29ecad8df07fbf93550b0bcdcbda25 2023-07-14T11:02:16.764 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-07-14T11:02:16.765 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout disabled 2023-07-14T11:02:16.781 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-07-14T11:02:16.782 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:02:16.782 INFO:teuthology.orchestra.run.smithi040.stdout:firewalld.service is not enabled 2023-07-14T11:02:16.782 INFO:teuthology.orchestra.run.smithi040.stdout:Not possible to enable service . firewalld.service is not available 2023-07-14T11:02:16.799 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-07-14T11:02:16.800 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout disabled 2023-07-14T11:02:16.816 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-07-14T11:02:16.816 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:02:16.817 INFO:teuthology.orchestra.run.smithi040.stdout:firewalld.service is not enabled 2023-07-14T11:02:16.817 INFO:teuthology.orchestra.run.smithi040.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2023-07-14T11:02:16.817 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mgr to start... 2023-07-14T11:02:16.817 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mgr... 2023-07-14T11:02:17.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:16 smithi040 systemd[1]: Started Ceph mgr.x for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:02:17.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:16.870+0000 7fd8b1ae4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:02:17.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:16.980+0000 7fd8b1ae4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:02:17.475 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:17.099+0000 7fd8b1ae4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:02:17.475 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:17.228+0000 7fd8b1ae4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:02:17.475 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:17.471+0000 7fd8b1ae4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:02:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:17 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1214813313' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-07-14T11:02:17.998 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:18.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:02:18.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:02:18.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "health": { 2023-07-14T11:02:18.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-07-14T11:02:18.000 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 0 2023-07-14T11:02:18.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "a" 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum_age": 3, 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "monmap": { 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-07-14T11:02:18.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-07-14T11:02:18.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-07-14T11:02:18.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:18.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-07-14T11:02:18.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-07-14T11:02:18.011 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-07-14T11:02:18.012 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-07-14T11:02:18.012 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-07-14T11:02:18.012 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-07-14T11:02:18.012 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.012 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-07-14T11:02:18.013 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-07-14T11:02:18.013 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-07-14T11:02:18.013 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-07-14T11:02:18.013 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-07-14T11:02:18.015 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "available": false, 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-07-14T11:02:18.016 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "modules": [ 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "iostat", 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "nfs", 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "restful" 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "services": {} 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-07-14T11:02:18.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "modified": "2023-07-14T11:02:08.999286+0000", 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "services": {} 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.385+0000 7fd29629e700 1 Processor -- start 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.385+0000 7fd29629e700 1 -- start start 2023-07-14T11:02:18.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.386+0000 7fd29629e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.386+0000 7fd29629e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd290105bd0 con 0x7fd290105280 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.386+0000 7fd28ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.386+0000 7fd28ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58888/0 (socket says 172.21.15.40:58888) 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.386+0000 7fd28ffff700 1 -- 172.21.15.40:0/4145052136 learned_addr learned my addr 172.21.15.40:0/4145052136 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.387+0000 7fd28ffff700 1 -- 172.21.15.40:0/4145052136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd290105d10 con 0x7fd290105280 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.387+0000 7fd28ffff700 1 --2- 172.21.15.40:0/4145052136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fd2800098a0 tx=0x7fd280009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8569ae5fa4c99a7f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:18.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.387+0000 7fd28effd700 1 -- 172.21.15.40:0/4145052136 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd280004160 con 0x7fd290105280 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.387+0000 7fd28effd700 1 -- 172.21.15.40:0/4145052136 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd2800042c0 con 0x7fd290105280 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.387+0000 7fd28effd700 1 -- 172.21.15.40:0/4145052136 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd280011a00 con 0x7fd290105280 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 -- 172.21.15.40:0/4145052136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 msgr2=0x7fd290105690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 --2- 172.21.15.40:0/4145052136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7fd2800098a0 tx=0x7fd280009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 -- 172.21.15.40:0/4145052136 shutdown_connections 2023-07-14T11:02:18.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 --2- 172.21.15.40:0/4145052136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd290105690 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 -- 172.21.15.40:0/4145052136 >> 172.21.15.40:0/4145052136 conn(0x7fd290100910 msgr2=0x7fd290102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 -- 172.21.15.40:0/4145052136 shutdown_connections 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.388+0000 7fd29629e700 1 -- 172.21.15.40:0/4145052136 wait complete. 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.389+0000 7fd29629e700 1 Processor -- start 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.389+0000 7fd29629e700 1 -- start start 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd29629e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:18.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd29629e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd29019b600 con 0x7fd290105280 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd28ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd28ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58892/0 (socket says 172.21.15.40:58892) 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd28ffff700 1 -- 172.21.15.40:0/1214813313 learned_addr learned my addr 172.21.15.40:0/1214813313 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.390+0000 7fd28ffff700 1 -- 172.21.15.40:0/1214813313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd280009550 con 0x7fd290105280 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd28ffff700 1 --2- 172.21.15.40:0/1214813313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fd280006af0 tx=0x7fd280011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd280003a40 con 0x7fd290105280 2023-07-14T11:02:18.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd280003ba0 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd29019b800 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd28001a6e0 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd29019bc20 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.391+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 660+0+0 (secure 0 0 0) 0x7fd28001ab60 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.392+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fd280019700 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.392+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd290061b40 con 0x7fd290105280 2023-07-14T11:02:18.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.397+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72459 (secure 0 0 0) 0x7fd280024a60 con 0x7fd290105280 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.474+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7fd2901070f0 con 0x7fd290105280 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.475+0000 7fd28d7fa700 1 -- 172.21.15.40:0/1214813313 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7fd280024c00 con 0x7fd290105280 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 msgr2=0x7fd29019b0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 --2- 172.21.15.40:0/1214813313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fd280006af0 tx=0x7fd280011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 shutdown_connections 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 --2- 172.21.15.40:0/1214813313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd290105280 0x7fd29019b0c0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:18.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 >> 172.21.15.40:0/1214813313 conn(0x7fd290100910 msgr2=0x7fd290101620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:18.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 shutdown_connections 2023-07-14T11:02:18.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:17.476+0000 7fd29629e700 1 -- 172.21.15.40:0/1214813313 wait complete. 2023-07-14T11:02:18.025 INFO:teuthology.orchestra.run.smithi040.stdout:mgr not available, waiting (1/15)... 2023-07-14T11:02:18.192 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:17.903+0000 7fd8b1ae4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:02:18.193 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:18.024+0000 7fd8b1ae4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:02:18.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:18.189+0000 7fd8b1ae4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:02:18.547 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:18.316+0000 7fd8b1ae4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:02:19.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:18.878+0000 7fd8b1ae4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:02:20.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:19.684+0000 7fd8b1ae4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:02:20.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:19.895+0000 7fd8b1ae4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:02:21.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:20.896+0000 7fd8b1ae4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:02:21.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:21.064+0000 7fd8b1ae4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:02:21.666 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:21.293+0000 7fd8b1ae4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:02:21.946 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:21.662+0000 7fd8b1ae4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:02:22.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:21.943+0000 7fd8b1ae4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:02:22.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:22.141+0000 7fd8b1ae4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:02:22.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:22.494+0000 7fd8b1ae4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:02:23.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:23.455+0000 7fd8b1ae4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:02:23.797 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:23.633+0000 7fd8b1ae4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:02:24.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:23.884+0000 7fd8b1ae4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:02:24.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:24.036+0000 7fd8b1ae4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:02:25.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:24.552+0000 7fd8b1ae4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:02:26.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:25.655+0000 7fd8b1ae4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:02:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: Activating manager daemon x 2023-07-14T11:02:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: mgrmap e2: x(active, starting, since 0.00455045s) 2023-07-14T11:02:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:02:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:02:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: Manager daemon x is now available 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' 2023-07-14T11:02:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' 2023-07-14T11:02:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:25 smithi040 ceph-mon[97954]: from='mgr.14100 172.21.15.40:0/98551879' entity='mgr.x' 2023-07-14T11:02:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:27 smithi040 ceph-mon[97954]: mgrmap e3: x(active, since 1.01052s) 2023-07-14T11:02:28.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "health": { 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-07-14T11:02:28.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 0 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "a" 2023-07-14T11:02:28.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "monmap": { 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.508 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-07-14T11:02:28.509 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:28.509 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-07-14T11:02:28.509 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-07-14T11:02:28.509 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-07-14T11:02:28.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-07-14T11:02:28.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-07-14T11:02:28.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-07-14T11:02:28.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-07-14T11:02:28.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-07-14T11:02:28.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-07-14T11:02:28.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-07-14T11:02:28.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:28.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "available": true, 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "modules": [ 2023-07-14T11:02:28.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "iostat", 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "nfs", 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "restful" 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ], 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "services": {} 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-07-14T11:02:28.519 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "modified": "2023-07-14T11:02:08.999286+0000", 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "services": {} 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout }, 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.763+0000 7fec68ce0700 1 Processor -- start 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.763+0000 7fec68ce0700 1 -- start start 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.763+0000 7fec68ce0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:28.520 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.763+0000 7fec68ce0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec6407c1c0 con 0x7fec6407bdb0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.764+0000 7fec6259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.764+0000 7fec6259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46152/0 (socket says 172.21.15.40:46152) 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.764+0000 7fec6259c700 1 -- 172.21.15.40:0/1240496399 learned_addr learned my addr 172.21.15.40:0/1240496399 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.764+0000 7fec6259c700 1 -- 172.21.15.40:0/1240496399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec6407a840 con 0x7fec6407bdb0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.764+0000 7fec6259c700 1 --2- 172.21.15.40:0/1240496399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fec540098a0 tx=0x7fec54009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bf870f7df9fac612 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec6159a700 1 -- 172.21.15.40:0/1240496399 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fec54004160 con 0x7fec6407bdb0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec6159a700 1 -- 172.21.15.40:0/1240496399 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fec540042c0 con 0x7fec6407bdb0 2023-07-14T11:02:28.521 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec6159a700 1 -- 172.21.15.40:0/1240496399 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fec54011a00 con 0x7fec6407bdb0 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec68ce0700 1 -- 172.21.15.40:0/1240496399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 msgr2=0x7fec6407a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec68ce0700 1 --2- 172.21.15.40:0/1240496399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fec540098a0 tx=0x7fec54009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec68ce0700 1 -- 172.21.15.40:0/1240496399 shutdown_connections 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec68ce0700 1 --2- 172.21.15.40:0/1240496399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6407a300 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.765+0000 7fec68ce0700 1 -- 172.21.15.40:0/1240496399 >> 172.21.15.40:0/1240496399 conn(0x7fec64100910 msgr2=0x7fec64102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.766+0000 7fec68ce0700 1 -- 172.21.15.40:0/1240496399 shutdown_connections 2023-07-14T11:02:28.522 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.766+0000 7fec68ce0700 1 -- 172.21.15.40:0/1240496399 wait complete. 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.766+0000 7fec68ce0700 1 Processor -- start 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec68ce0700 1 -- start start 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec68ce0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec68ce0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec6419b6a0 con 0x7fec6407bdb0 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec6259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec6259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46154/0 (socket says 172.21.15.40:46154) 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.767+0000 7fec6259c700 1 -- 172.21.15.40:0/913334052 learned_addr learned my addr 172.21.15.40:0/913334052 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:28.523 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec6259c700 1 -- 172.21.15.40:0/913334052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec54009550 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec6259c700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fec54006af0 tx=0x7fec54011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fec54003a40 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fec54003ba0 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fec5401a700 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec6419b8a0 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.768+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec6419bcc0 con 0x7fec6407bdb0 2023-07-14T11:02:28.524 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.769+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 4) v1 ==== 45472+0+0 (secure 0 0 0) 0x7fec54024ad0 con 0x7fec6407bdb0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.769+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec64061b40 con 0x7fec6407bdb0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.770+0000 7fec537fe700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 0x7fec4c03abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.770+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fec5404c630 con 0x7fec6407bdb0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.770+0000 7fec61d9b700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 0x7fec4c03abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.771+0000 7fec61d9b700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 0x7fec4c03abc0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fec58006fd0 tx=0x7fec58006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:27.776+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fec54018bd0 con 0x7fec6407bdb0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.062+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7fec6419ee40 con 0x7fec6407bdb0 2023-07-14T11:02:28.525 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.063+0000 7fec537fe700 1 -- 172.21.15.40:0/913334052 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1238 (secure 0 0 0) 0x7fec540187c0 con 0x7fec6407bdb0 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 msgr2=0x7fec4c03abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 0x7fec4c03abc0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7fec58006fd0 tx=0x7fec58006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 msgr2=0x7fec6419b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fec54006af0 tx=0x7fec54011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 shutdown_connections 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fec4c038710 0x7fec4c03abc0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 --2- 172.21.15.40:0/913334052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec6407bdb0 0x7fec6419b160 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:28.526 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.065+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 >> 172.21.15.40:0/913334052 conn(0x7fec64100910 msgr2=0x7fec64101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:28.527 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.066+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 shutdown_connections 2023-07-14T11:02:28.527 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:28.066+0000 7fec68ce0700 1 -- 172.21.15.40:0/913334052 wait complete. 2023-07-14T11:02:28.527 INFO:teuthology.orchestra.run.smithi040.stdout:mgr is available 2023-07-14T11:02:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:28 smithi040 ceph-mon[97954]: mgrmap e4: x(active, since 2s) 2023-07-14T11:02:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:28 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/913334052' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-07-14T11:02:29.797 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:29.798 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [global] 2023-07-14T11:02:29.798 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout fsid = bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:29.798 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-07-14T11:02:29.798 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [mgr] 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-07-14T11:02:29.799 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout [osd] 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-07-14T11:02:29.800 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.002+0000 7f8df2490700 1 Processor -- start 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.002+0000 7f8df2490700 1 -- start start 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.002+0000 7f8df2490700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.002+0000 7f8df2490700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dec07a860 con 0x7f8dec07bdd0 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.003+0000 7f8debfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.003+0000 7f8debfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46170/0 (socket says 172.21.15.40:46170) 2023-07-14T11:02:29.801 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.003+0000 7f8debfff700 1 -- 172.21.15.40:0/2278039209 learned_addr learned my addr 172.21.15.40:0/2278039209 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.003+0000 7f8debfff700 1 -- 172.21.15.40:0/2278039209 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8dec07a9a0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.003+0000 7f8debfff700 1 --2- 172.21.15.40:0/2278039209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc0098a0 tx=0x7f8ddc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bead74a0e0d4ea51 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8deaffd700 1 -- 172.21.15.40:0/2278039209 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8ddc004160 con 0x7f8dec07bdd0 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8deaffd700 1 -- 172.21.15.40:0/2278039209 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8ddc0042c0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8deaffd700 1 -- 172.21.15.40:0/2278039209 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8ddc011a00 con 0x7f8dec07bdd0 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 -- 172.21.15.40:0/2278039209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 msgr2=0x7f8dec07a320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:29.802 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 --2- 172.21.15.40:0/2278039209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc0098a0 tx=0x7f8ddc009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 -- 172.21.15.40:0/2278039209 shutdown_connections 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 --2- 172.21.15.40:0/2278039209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec07a320 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 -- 172.21.15.40:0/2278039209 >> 172.21.15.40:0/2278039209 conn(0x7f8dec100910 msgr2=0x7f8dec102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 -- 172.21.15.40:0/2278039209 shutdown_connections 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.004+0000 7f8df2490700 1 -- 172.21.15.40:0/2278039209 wait complete. 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.005+0000 7f8df2490700 1 Processor -- start 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8df2490700 1 -- start start 2023-07-14T11:02:29.803 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8df2490700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:29.804 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8df2490700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8dec19b660 con 0x7f8dec07bdd0 2023-07-14T11:02:29.804 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8debfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:29.804 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8debfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46186/0 (socket says 172.21.15.40:46186) 2023-07-14T11:02:29.804 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8debfff700 1 -- 172.21.15.40:0/3260802096 learned_addr learned my addr 172.21.15.40:0/3260802096 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:29.805 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.006+0000 7f8debfff700 1 -- 172.21.15.40:0/3260802096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ddc009550 con 0x7f8dec07bdd0 2023-07-14T11:02:29.805 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8debfff700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc006af0 tx=0x7f8ddc011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:29.805 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8ddc003a40 con 0x7f8dec07bdd0 2023-07-14T11:02:29.805 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8ddc003ba0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.805 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f8ddc01a700 con 0x7f8dec07bdd0 2023-07-14T11:02:29.806 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8dec19b860 con 0x7f8dec07bdd0 2023-07-14T11:02:29.806 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.007+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8dec19bc80 con 0x7f8dec07bdd0 2023-07-14T11:02:29.806 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.008+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 4) v1 ==== 45472+0+0 (secure 0 0 0) 0x7f8ddc024ad0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.806 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.008+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8dec10b560 con 0x7f8dec07bdd0 2023-07-14T11:02:29.807 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.009+0000 7f8de97fa700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 0x7f8dd403abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:29.807 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.009+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f8ddc04c600 con 0x7f8dec07bdd0 2023-07-14T11:02:29.808 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.009+0000 7f8deb7fe700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 0x7f8dd403abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:29.808 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.009+0000 7f8deb7fe700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 0x7f8dd403abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f8de0006fd0 tx=0x7f8de0006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:29.808 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.013+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8ddc018ba0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.809 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.218+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f8dec10b720 con 0x7f8dec07bdd0 2023-07-14T11:02:29.809 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.219+0000 7f8de97fa700 1 -- 172.21.15.40:0/3260802096 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) v1 ==== 70+0+373 (secure 0 0 0) 0x7f8ddc0187c0 con 0x7f8dec07bdd0 2023-07-14T11:02:29.809 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 msgr2=0x7f8dd403abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 0x7f8dd403abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f8de0006fd0 tx=0x7f8de0006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 msgr2=0x7f8dec19b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f8ddc006af0 tx=0x7f8ddc011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 shutdown_connections 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f8dd4038710 0x7f8dd403abc0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 --2- 172.21.15.40:0/3260802096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8dec07bdd0 0x7f8dec19b120 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 >> 172.21.15.40:0/3260802096 conn(0x7f8dec100910 msgr2=0x7f8dec1016c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:29.810 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 shutdown_connections 2023-07-14T11:02:29.811 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:29.222+0000 7f8df2490700 1 -- 172.21.15.40:0/3260802096 wait complete. 2023-07-14T11:02:29.811 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling cephadm module... 2023-07-14T11:02:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:29 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3260802096' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-07-14T11:02:30.947 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:30 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2464211804' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-07-14T11:02:30.947 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setuser ceph since I am not root 2023-07-14T11:02:30.948 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setgroup ceph since I am not root 2023-07-14T11:02:30.948 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:30.944+0000 7fa0b80c2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:02:31.174 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.353+0000 7fc92ae2e700 1 Processor -- start 2023-07-14T11:02:31.174 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.353+0000 7fc92ae2e700 1 -- start start 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.353+0000 7fc92ae2e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.353+0000 7fc92ae2e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc92407c1a0 con 0x7fc92407bd90 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc928bca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc928bca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46198/0 (socket says 172.21.15.40:46198) 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc928bca700 1 -- 172.21.15.40:0/2151294169 learned_addr learned my addr 172.21.15.40:0/2151294169 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc928bca700 1 -- 172.21.15.40:0/2151294169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc92407a820 con 0x7fc92407bd90 2023-07-14T11:02:31.175 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc928bca700 1 --2- 172.21.15.40:0/2151294169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fc9180098a0 tx=0x7fc918009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=70b6e4c52bf0abce server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc9237fe700 1 -- 172.21.15.40:0/2151294169 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc918004160 con 0x7fc92407bd90 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.354+0000 7fc9237fe700 1 -- 172.21.15.40:0/2151294169 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc9180042c0 con 0x7fc92407bd90 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc9237fe700 1 -- 172.21.15.40:0/2151294169 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc918011a00 con 0x7fc92407bd90 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2151294169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 msgr2=0x7fc92407a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2151294169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fc9180098a0 tx=0x7fc918009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2151294169 shutdown_connections 2023-07-14T11:02:31.176 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2151294169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92407a2e0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2151294169 >> 172.21.15.40:0/2151294169 conn(0x7fc924100910 msgr2=0x7fc924102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2151294169 shutdown_connections 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.355+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2151294169 wait complete. 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.356+0000 7fc92ae2e700 1 Processor -- start 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.356+0000 7fc92ae2e700 1 -- start start 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc92ae2e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:31.177 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc92ae2e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc92419b660 con 0x7fc92407bd90 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc928bca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc928bca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46200/0 (socket says 172.21.15.40:46200) 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc928bca700 1 -- 172.21.15.40:0/2464211804 learned_addr learned my addr 172.21.15.40:0/2464211804 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc928bca700 1 -- 172.21.15.40:0/2464211804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc918009550 con 0x7fc92407bd90 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.357+0000 7fc928bca700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fc918006b20 tx=0x7fc918011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:31.178 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.358+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc918003b70 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.358+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc918003cd0 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.358+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc9180247c0 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.358+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc92419b860 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.358+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc92419bc80 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.359+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc924061b40 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.359+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 4) v1 ==== 45472+0+0 (secure 0 0 0) 0x7fc91801ab40 con 0x7fc92407bd90 2023-07-14T11:02:31.179 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.359+0000 7fc921ffb700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 0x7fc91403abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.360+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fc91804c510 con 0x7fc92407bd90 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.360+0000 7fc923fff700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 0x7fc91403abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.360+0000 7fc923fff700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 0x7fc91403abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc910006fd0 tx=0x7fc910006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.365+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc91801a950 con 0x7fc92407bd90 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.631+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) v1 -- 0x7fc92419ee00 con 0x7fc92407bd90 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.681+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 5) v1 ==== 45483+0+0 (secure 0 0 0) 0x7fc91801a430 con 0x7fc92407bd90 2023-07-14T11:02:31.180 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.683+0000 7fc921ffb700 1 -- 172.21.15.40:0/2464211804 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) v1 ==== 86+0+0 (secure 0 0 0) 0x7fc91804c270 con 0x7fc92407bd90 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 msgr2=0x7fc91403abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 0x7fc91403abc0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc910006fd0 tx=0x7fc910006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 msgr2=0x7fc92419b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fc918006b20 tx=0x7fc918011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 shutdown_connections 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7fc914038710 0x7fc91403abc0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:31.181 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 --2- 172.21.15.40:0/2464211804 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc92407bd90 0x7fc92419b120 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:31.182 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.685+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 >> 172.21.15.40:0/2464211804 conn(0x7fc924100910 msgr2=0x7fc9241016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:31.182 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.686+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 shutdown_connections 2023-07-14T11:02:31.182 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:30.686+0000 7fc92ae2e700 1 -- 172.21.15.40:0/2464211804 wait complete. 2023-07-14T11:02:31.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:31.054+0000 7fa0b80c2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:02:31.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:31.173+0000 7fa0b80c2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:02:31.683 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:31.306+0000 7fa0b80c2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:02:31.683 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:31.549+0000 7fa0b80c2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:02:31.964 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2464211804' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-07-14T11:02:31.964 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-mon[97954]: mgrmap e5: x(active, since 5s) 2023-07-14T11:02:32.259 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:31.961+0000 7fa0b80c2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:02:32.259 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:32.086+0000 7fa0b80c2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:02:32.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:32.256+0000 7fa0b80c2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:02:32.547 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:32.384+0000 7fa0b80c2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:02:32.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:02:32.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "available": true, 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.704+0000 7f4eedda1700 1 Processor -- start 2023-07-14T11:02:32.553 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4eedda1700 1 -- start start 2023-07-14T11:02:32.555 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4eedda1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:32.556 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4eedda1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ee8105bd0 con 0x7f4ee8105280 2023-07-14T11:02:32.556 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4ee77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:32.556 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4ee77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46232/0 (socket says 172.21.15.40:46232) 2023-07-14T11:02:32.556 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.705+0000 7f4ee77fe700 1 -- 172.21.15.40:0/417587710 learned_addr learned my addr 172.21.15.40:0/417587710 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:32.556 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.706+0000 7f4ee77fe700 1 -- 172.21.15.40:0/417587710 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ee8105d10 con 0x7f4ee8105280 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.706+0000 7f4ee77fe700 1 --2- 172.21.15.40:0/417587710 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4ed80098a0 tx=0x7f4ed8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=8521fc37dfe5a2a5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.706+0000 7f4ee67fc700 1 -- 172.21.15.40:0/417587710 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4ed8004160 con 0x7f4ee8105280 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.706+0000 7f4ee67fc700 1 -- 172.21.15.40:0/417587710 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4ed80042c0 con 0x7f4ee8105280 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.706+0000 7f4ee67fc700 1 -- 172.21.15.40:0/417587710 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4ed8011a00 con 0x7f4ee8105280 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 -- 172.21.15.40:0/417587710 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 msgr2=0x7f4ee8105690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:32.557 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 --2- 172.21.15.40:0/417587710 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4ed80098a0 tx=0x7f4ed8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 -- 172.21.15.40:0/417587710 shutdown_connections 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 --2- 172.21.15.40:0/417587710 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8105690 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 -- 172.21.15.40:0/417587710 >> 172.21.15.40:0/417587710 conn(0x7f4ee8100910 msgr2=0x7f4ee8102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 -- 172.21.15.40:0/417587710 shutdown_connections 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.707+0000 7f4eedda1700 1 -- 172.21.15.40:0/417587710 wait complete. 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.709+0000 7f4eedda1700 1 Processor -- start 2023-07-14T11:02:32.558 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.709+0000 7f4eedda1700 1 -- start start 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.709+0000 7f4eedda1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.709+0000 7f4eedda1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ee8199400 con 0x7f4ee8105280 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.710+0000 7f4ee77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.710+0000 7f4ee77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46234/0 (socket says 172.21.15.40:46234) 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.710+0000 7f4ee77fe700 1 -- 172.21.15.40:0/217340030 learned_addr learned my addr 172.21.15.40:0/217340030 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.710+0000 7f4ee77fe700 1 -- 172.21.15.40:0/217340030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ed8009550 con 0x7f4ee8105280 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.710+0000 7f4ee77fe700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4ed8006af0 tx=0x7f4ed8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:32.559 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.711+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4ed8003a40 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.711+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ee8199600 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.711+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4ed8003ba0 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.711+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4ed801a6e0 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.711+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ee8199a20 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.712+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4ee8061b40 con 0x7f4ee8105280 2023-07-14T11:02:32.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.712+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 5) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f4ed8024430 con 0x7f4ee8105280 2023-07-14T11:02:32.563 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.712+0000 7f4ee4ff9700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 0x7f4ed003ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:32.563 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.713+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f4ed804c6e0 con 0x7f4ee8105280 2023-07-14T11:02:32.563 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.713+0000 7f4ee6ffd700 1 -- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 msgr2=0x7f4ed003ac10 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:32.563 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.713+0000 7f4ee6ffd700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 0x7f4ed003ac10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:02:32.563 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.717+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4ed80246e0 con 0x7f4ee8105280 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.914+0000 7f4ee6ffd700 1 -- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 msgr2=0x7f4ed003ac10 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.914+0000 7f4ee6ffd700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 0x7f4ed003ac10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.984+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f4ee819c880 con 0x7f4ee8105280 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.984+0000 7f4ee4ff9700 1 -- 172.21.15.40:0/217340030 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) v1 ==== 56+0+88 (secure 0 0 0) 0x7f4ed8018350 con 0x7f4ee8105280 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 msgr2=0x7f4ed003ac10 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 0x7f4ed003ac10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 msgr2=0x7f4ee8198ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:32.564 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f4ed8006af0 tx=0x7f4ed8011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 shutdown_connections 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f4ed0038760 0x7f4ed003ac10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 --2- 172.21.15.40:0/217340030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ee8105280 0x7f4ee8198ec0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 >> 172.21.15.40:0/217340030 conn(0x7f4ee8100910 msgr2=0x7f4ee8101620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 shutdown_connections 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:31.987+0000 7f4eedda1700 1 -- 172.21.15.40:0/217340030 wait complete. 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for the mgr to restart... 2023-07-14T11:02:32.565 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mgr epoch 5... 2023-07-14T11:02:32.936 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:32 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/217340030' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-07-14T11:02:33.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:32.975+0000 7fa0b80c2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:02:34.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:33.751+0000 7fa0b80c2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:02:34.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:33.964+0000 7fa0b80c2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:02:35.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:34 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:34.942+0000 7fa0b80c2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:02:35.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:35 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:35.066+0000 7fa0b80c2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:02:35.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:35 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:35.293+0000 7fa0b80c2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:02:35.944 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:35 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:35.662+0000 7fa0b80c2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:02:36.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:35 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:35.941+0000 7fa0b80c2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:02:36.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:36 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:36.140+0000 7fa0b80c2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:02:36.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:36 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:36.492+0000 7fa0b80c2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:02:37.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:37 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:37.468+0000 7fa0b80c2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:02:37.797 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:37 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:37.647+0000 7fa0b80c2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:02:38.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:37 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:37.901+0000 7fa0b80c2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:02:38.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:38 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:38.053+0000 7fa0b80c2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:02:38.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:38 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:38.336+0000 7fa0b80c2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:02:39.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:39 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:39.439+0000 7fa0b80c2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:02:40.481 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: Active manager daemon x restarted 2023-07-14T11:02:40.481 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: Activating manager daemon x 2023-07-14T11:02:40.481 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: osdmap e2: 0 total, 0 up, 0 in 2023-07-14T11:02:40.481 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: mgrmap e6: x(active, starting, since 0.0039089s) 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: Manager daemon x is now available 2023-07-14T11:02:40.482 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: Found migration_current of "None". Setting to last migration. 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:02:40.483 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:40.484 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:40 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:40.980 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:02:40.980 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-07-14T11:02:40.980 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "initialized": true 2023-07-14T11:02:40.980 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.147+0000 7f0825991700 1 Processor -- start 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f0825991700 1 -- start start 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f0825991700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f0825991700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0820105bd0 con 0x7f0820105280 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f081effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f081effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46238/0 (socket says 172.21.15.40:46238) 2023-07-14T11:02:40.981 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.148+0000 7f081effd700 1 -- 172.21.15.40:0/4237512205 learned_addr learned my addr 172.21.15.40:0/4237512205 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.149+0000 7f081effd700 1 -- 172.21.15.40:0/4237512205 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0820105d10 con 0x7f0820105280 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.149+0000 7f081effd700 1 --2- 172.21.15.40:0/4237512205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f08100098a0 tx=0x7f0810009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f39156b4476539a7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.149+0000 7f081dffb700 1 -- 172.21.15.40:0/4237512205 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0810004160 con 0x7f0820105280 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.149+0000 7f081dffb700 1 -- 172.21.15.40:0/4237512205 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f08100042c0 con 0x7f0820105280 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.149+0000 7f081dffb700 1 -- 172.21.15.40:0/4237512205 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0810011a00 con 0x7f0820105280 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 -- 172.21.15.40:0/4237512205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 msgr2=0x7f0820105690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 --2- 172.21.15.40:0/4237512205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f08100098a0 tx=0x7f0810009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.982 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 -- 172.21.15.40:0/4237512205 shutdown_connections 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 --2- 172.21.15.40:0/4237512205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f0820105690 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 -- 172.21.15.40:0/4237512205 >> 172.21.15.40:0/4237512205 conn(0x7f0820100910 msgr2=0x7f0820102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 -- 172.21.15.40:0/4237512205 shutdown_connections 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.150+0000 7f0825991700 1 -- 172.21.15.40:0/4237512205 wait complete. 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.151+0000 7f0825991700 1 Processor -- start 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.151+0000 7f0825991700 1 -- start start 2023-07-14T11:02:40.983 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f0825991700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f0825991700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f082019b600 con 0x7f0820105280 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f081effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f081effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46250/0 (socket says 172.21.15.40:46250) 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f081effd700 1 -- 172.21.15.40:0/4074091793 learned_addr learned my addr 172.21.15.40:0/4074091793 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f081effd700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0810009550 con 0x7f0820105280 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.152+0000 7f081effd700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f0810006af0 tx=0x7f0810011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.153+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0810003a40 con 0x7f0820105280 2023-07-14T11:02:40.984 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.153+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f082019b800 con 0x7f0820105280 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.153+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0810003ba0 con 0x7f0820105280 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.153+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f081001a6e0 con 0x7f0820105280 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.153+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f082019bc20 con 0x7f0820105280 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.154+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 5) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f0810024a60 con 0x7f0820105280 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.154+0000 7f082498f700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.154+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f080803b2d0 con 0x7f0808038710 2023-07-14T11:02:40.985 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.154+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f081004c760 con 0x7f0820105280 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.155+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.155+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.356+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.356+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.757+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:33.757+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:34.558+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.986 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:34.558+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:36.159+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:36.159+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:39.363+0000 7f081e7fc700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/137331595 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:39.363+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 6.400000 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:39.444+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mgrmap(e 6) v1 ==== 45250+0+0 (secure 0 0 0) 0x7f0810024610 con 0x7f0820105280 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:39.444+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 msgr2=0x7f080803abc0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:39.444+0000 7f082498f700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.987 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.446+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 7) v1 ==== 45377+0+0 (secure 0 0 0) 0x7f081004e2b0 con 0x7f0820105280 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.446+0000 7f082498f700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 0x7f080803e3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.446+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f080803b2d0 con 0x7f080803bfe0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.446+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 0x7f080803e3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.447+0000 7f081e7fc700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 0x7f080803e3c0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f0814003a10 tx=0x7f0814009290 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.448+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6910 (secure 0 0 0) 0x7f080803b2d0 con 0x7f080803bfe0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.455+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f082019c400 con 0x7f080803bfe0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f082498f700 1 -- 172.21.15.40:0/4074091793 <== mgr.14116 v2:172.21.15.40:6800/2376955220 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+51 (secure 0 0 0) 0x7f082019c400 con 0x7f080803bfe0 2023-07-14T11:02:40.988 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 msgr2=0x7f080803e3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 0x7f080803e3c0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f0814003a10 tx=0x7f0814009290 comp rx=0 tx=0).stop 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 msgr2=0x7f082019b0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f0810006af0 tx=0x7f0810011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 shutdown_connections 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f080803bfe0 0x7f080803e3c0 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0820105280 0x7f082019b0c0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.989 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 --2- 172.21.15.40:0/4074091793 >> [v2:172.21.15.40:6800/137331595,v1:172.21.15.40:6801/137331595] conn(0x7f0808038710 0x7f080803abc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:40.990 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 >> 172.21.15.40:0/4074091793 conn(0x7f0820100910 msgr2=0x7f0820101620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:40.990 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.456+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 shutdown_connections 2023-07-14T11:02:40.990 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:40.457+0000 7f0825991700 1 -- 172.21.15.40:0/4074091793 wait complete. 2023-07-14T11:02:40.990 INFO:teuthology.orchestra.run.smithi040.stdout:mgr epoch 5 is available 2023-07-14T11:02:40.990 INFO:teuthology.orchestra.run.smithi040.stdout:Setting orchestrator backend to cephadm... 2023-07-14T11:02:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:41 smithi040 ceph-mon[97954]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-07-14T11:02:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:41 smithi040 ceph-mon[97954]: mgrmap e7: x(active, since 1.00683s) 2023-07-14T11:02:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:41 smithi040 ceph-mon[97954]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-07-14T11:02:41.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:41 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:42.259 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.520+0000 7f510d093700 1 Processor -- start 2023-07-14T11:02:42.265 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.521+0000 7f510d093700 1 -- start start 2023-07-14T11:02:42.265 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.521+0000 7f510d093700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:42.265 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.521+0000 7f510d093700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f510807a880 con 0x7f510807bdf0 2023-07-14T11:02:42.265 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5106d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:42.265 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5106d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:33430/0 (socket says 172.21.15.40:33430) 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5106d9d700 1 -- 172.21.15.40:0/3819499515 learned_addr learned my addr 172.21.15.40:0/3819499515 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5106d9d700 1 -- 172.21.15.40:0/3819499515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f510807a9c0 con 0x7f510807bdf0 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5106d9d700 1 --2- 172.21.15.40:0/3819499515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f50f80098a0 tx=0x7f50f8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=656d580445a7f27 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5105d9b700 1 -- 172.21.15.40:0/3819499515 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f50f8004160 con 0x7f510807bdf0 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5105d9b700 1 -- 172.21.15.40:0/3819499515 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f50f80042c0 con 0x7f510807bdf0 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.522+0000 7f5105d9b700 1 -- 172.21.15.40:0/3819499515 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f50f8011a00 con 0x7f510807bdf0 2023-07-14T11:02:42.266 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 -- 172.21.15.40:0/3819499515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 msgr2=0x7f510807a340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 --2- 172.21.15.40:0/3819499515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f50f80098a0 tx=0x7f50f8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 -- 172.21.15.40:0/3819499515 shutdown_connections 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 --2- 172.21.15.40:0/3819499515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510807a340 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 -- 172.21.15.40:0/3819499515 >> 172.21.15.40:0/3819499515 conn(0x7f5108100910 msgr2=0x7f5108102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 -- 172.21.15.40:0/3819499515 shutdown_connections 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.523+0000 7f510d093700 1 -- 172.21.15.40:0/3819499515 wait complete. 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.525+0000 7f510d093700 1 Processor -- start 2023-07-14T11:02:42.267 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.525+0000 7f510d093700 1 -- start start 2023-07-14T11:02:42.268 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.525+0000 7f510d093700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:42.268 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.525+0000 7f510d093700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f510819b670 con 0x7f510807bdf0 2023-07-14T11:02:42.268 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f5106d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:42.268 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f5106d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:33434/0 (socket says 172.21.15.40:33434) 2023-07-14T11:02:42.269 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f5106d9d700 1 -- 172.21.15.40:0/261383587 learned_addr learned my addr 172.21.15.40:0/261383587 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:42.269 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f5106d9d700 1 -- 172.21.15.40:0/261383587 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f50f8009550 con 0x7f510807bdf0 2023-07-14T11:02:42.269 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f5106d9d700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f50f8006af0 tx=0x7f50f8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:42.269 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f50f8003a40 con 0x7f510807bdf0 2023-07-14T11:02:42.270 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f50f8003ba0 con 0x7f510807bdf0 2023-07-14T11:02:42.270 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.526+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f510819b870 con 0x7f510807bdf0 2023-07-14T11:02:42.270 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.527+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f50f801a700 con 0x7f510807bdf0 2023-07-14T11:02:42.271 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.527+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f510819bc90 con 0x7f510807bdf0 2023-07-14T11:02:42.271 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.528+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f50f8024430 con 0x7f510807bdf0 2023-07-14T11:02:42.271 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.528+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5108061b40 con 0x7f510807bdf0 2023-07-14T11:02:42.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.528+0000 7f50effff700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 0x7f50f003abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:42.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.528+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f50f804c7f0 con 0x7f510807bdf0 2023-07-14T11:02:42.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.528+0000 7f510659c700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 0x7f50f003abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:42.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.529+0000 7f510659c700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 0x7f50f003abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f50fc006fd0 tx=0x7f50fc006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:42.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.534+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f50f8018920 con 0x7f510807bdf0 2023-07-14T11:02:42.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.756+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) v1 -- 0x7f5108101db0 con 0x7f50f0038710 2023-07-14T11:02:42.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.764+0000 7f50effff700 1 -- 172.21.15.40:0/261383587 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f5108101db0 con 0x7f50f0038710 2023-07-14T11:02:42.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 msgr2=0x7f50f003abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:42.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 0x7f50f003abc0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f50fc006fd0 tx=0x7f50fc006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:42.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 msgr2=0x7f510819b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:42.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f50f8006af0 tx=0x7f50f8011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:42.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 shutdown_connections 2023-07-14T11:02:42.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f50f0038710 0x7f50f003abc0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:42.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 --2- 172.21.15.40:0/261383587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f510807bdf0 0x7f510819b130 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:42.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.766+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 >> 172.21.15.40:0/261383587 conn(0x7f5108100910 msgr2=0x7f51081016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:42.276 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.767+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 shutdown_connections 2023-07-14T11:02:42.276 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:41.767+0000 7f510d093700 1 -- 172.21.15.40:0/261383587 wait complete. 2023-07-14T11:02:42.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: [14/Jul/2023:11:02:40] ENGINE Bus STARTING 2023-07-14T11:02:42.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: [14/Jul/2023:11:02:41] ENGINE Serving on http://172.21.15.40:8765 2023-07-14T11:02:42.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: [14/Jul/2023:11:02:41] ENGINE Serving on https://172.21.15.40:7150 2023-07-14T11:02:42.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: [14/Jul/2023:11:02:41] ENGINE Bus STARTED 2023-07-14T11:02:42.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: mgrmap e8: x(active, since 2s) 2023-07-14T11:02:42.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:42.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:42.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:42 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:43.545 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout value unchanged 2023-07-14T11:02:43.545 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.824+0000 7fb803e50700 1 Processor -- start 2023-07-14T11:02:43.545 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.824+0000 7fb803e50700 1 -- start start 2023-07-14T11:02:43.545 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.825+0000 7fb803e50700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.825+0000 7fb803e50700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7fc07c1a0 con 0x7fb7fc07bd90 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.825+0000 7fb801bec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.825+0000 7fb801bec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:33446/0 (socket says 172.21.15.40:33446) 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.825+0000 7fb801bec700 1 -- 172.21.15.40:0/1860468444 learned_addr learned my addr 172.21.15.40:0/1860468444 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.826+0000 7fb801bec700 1 -- 172.21.15.40:0/1860468444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7fc07a820 con 0x7fb7fc07bd90 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.826+0000 7fb801bec700 1 --2- 172.21.15.40:0/1860468444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fb7f80098a0 tx=0x7fb7f8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9f1e880f9e75f363 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:43.546 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.826+0000 7fb800bea700 1 -- 172.21.15.40:0/1860468444 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7f8004160 con 0x7fb7fc07bd90 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.826+0000 7fb800bea700 1 -- 172.21.15.40:0/1860468444 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb7f80042c0 con 0x7fb7fc07bd90 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.826+0000 7fb800bea700 1 -- 172.21.15.40:0/1860468444 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7f8011a00 con 0x7fb7fc07bd90 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 -- 172.21.15.40:0/1860468444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 msgr2=0x7fb7fc07a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 --2- 172.21.15.40:0/1860468444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fb7f80098a0 tx=0x7fb7f8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 -- 172.21.15.40:0/1860468444 shutdown_connections 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 --2- 172.21.15.40:0/1860468444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc07a2e0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 -- 172.21.15.40:0/1860468444 >> 172.21.15.40:0/1860468444 conn(0x7fb7fc100910 msgr2=0x7fb7fc102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:43.547 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 -- 172.21.15.40:0/1860468444 shutdown_connections 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.827+0000 7fb803e50700 1 -- 172.21.15.40:0/1860468444 wait complete. 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb803e50700 1 Processor -- start 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb803e50700 1 -- start start 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb803e50700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb803e50700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb7fc19b660 con 0x7fb7fc07bd90 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb801bec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:43.548 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.828+0000 7fb801bec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:33462/0 (socket says 172.21.15.40:33462) 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb801bec700 1 -- 172.21.15.40:0/3759499968 learned_addr learned my addr 172.21.15.40:0/3759499968 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb801bec700 1 -- 172.21.15.40:0/3759499968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb7f8009550 con 0x7fb7fc07bd90 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb801bec700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fb7f8006b20 tx=0x7fb7f8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7f8003b70 con 0x7fb7fc07bd90 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb7f8003cd0 con 0x7fb7fc07bd90 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb7fc19b860 con 0x7fb7fc07bd90 2023-07-14T11:02:43.549 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.829+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb7f80247c0 con 0x7fb7fc07bd90 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.830+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7fc19bc80 con 0x7fb7fc07bd90 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.830+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7fb7f801a430 con 0x7fb7fc07bd90 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.830+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb7fc061b40 con 0x7fb7fc07bd90 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.831+0000 7fb7f2ffd700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 0x7fb7e803ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.831+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fb7f804c970 con 0x7fb7fc07bd90 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.831+0000 7fb8013eb700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 0x7fb7e803ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.832+0000 7fb8013eb700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 0x7fb7e803ac10 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb7ec006fd0 tx=0x7fb7ec006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:43.550 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:42.836+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb7f801a6e0 con 0x7fb7fc07bd90 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.043+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) v1 -- 0x7fb7fc101db0 con 0x7fb7e8038760 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.044+0000 7fb7f2ffd700 1 -- 172.21.15.40:0/3759499968 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+16 (secure 0 0 0) 0x7fb7fc101db0 con 0x7fb7e8038760 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 msgr2=0x7fb7e803ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 0x7fb7e803ac10 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fb7ec006fd0 tx=0x7fb7ec006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 msgr2=0x7fb7fc19b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fb7f8006b20 tx=0x7fb7f8011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:43.551 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 shutdown_connections 2023-07-14T11:02:43.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb7e8038760 0x7fb7e803ac10 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:43.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 --2- 172.21.15.40:0/3759499968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7fc07bd90 0x7fb7fc19b120 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:43.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 >> 172.21.15.40:0/3759499968 conn(0x7fb7fc100910 msgr2=0x7fb7fc1016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:43.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 shutdown_connections 2023-07-14T11:02:43.552 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:43.047+0000 7fb803e50700 1 -- 172.21.15.40:0/3759499968 wait complete. 2023-07-14T11:02:43.553 INFO:teuthology.orchestra.run.smithi040.stdout:Generating ssh key... 2023-07-14T11:02:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:43 smithi040 ceph-mon[97954]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:45.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: Generating public/private rsa key pair. 2023-07-14T11:02:45.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: Your identification has been saved in /tmp/tmpdbx8ps9p/key. 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: Your public key has been saved in /tmp/tmpdbx8ps9p/key.pub. 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: The key fingerprint is: 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: SHA256:FbPpRNIt7OpPFlqnIDT0LADEIIv+P6xBdkZIYoFQGE8 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: The key's randomart image is: 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: +---[RSA 3072]----+ 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: |B@Eo. . .o+. | 2023-07-14T11:02:45.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: |*++ .o o o+=. | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: |o .. .+ o.=. | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: |. .. o +. | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: | . o o. S.+ . | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: | + o ..+ + | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: | o. .. + | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: | oo .o | 2023-07-14T11:02:45.048 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: | .... .. | 2023-07-14T11:02:45.049 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:44 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: +----[SHA256]-----+ 2023-07-14T11:02:45.225 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.091+0000 7fb2011e9700 1 Processor -- start 2023-07-14T11:02:45.225 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.091+0000 7fb2011e9700 1 -- start start 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.091+0000 7fb2011e9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.091+0000 7fb2011e9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1fc07a860 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.091+0000 7fb1fad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.092+0000 7fb1fad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59932/0 (socket says 172.21.15.40:59932) 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.092+0000 7fb1fad9d700 1 -- 172.21.15.40:0/680100361 learned_addr learned my addr 172.21.15.40:0/680100361 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:45.226 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.092+0000 7fb1fad9d700 1 -- 172.21.15.40:0/680100361 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb1fc07a9a0 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.092+0000 7fb1fad9d700 1 --2- 172.21.15.40:0/680100361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb1ec0098a0 tx=0x7fb1ec009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ba907e835e84a912 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.092+0000 7fb1f9d9b700 1 -- 172.21.15.40:0/680100361 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb1ec004160 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb1f9d9b700 1 -- 172.21.15.40:0/680100361 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb1ec0042c0 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb1f9d9b700 1 -- 172.21.15.40:0/680100361 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb1ec011a00 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 -- 172.21.15.40:0/680100361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 msgr2=0x7fb1fc07a320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 --2- 172.21.15.40:0/680100361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb1ec0098a0 tx=0x7fb1ec009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:45.227 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 -- 172.21.15.40:0/680100361 shutdown_connections 2023-07-14T11:02:45.228 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 --2- 172.21.15.40:0/680100361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc07a320 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:45.228 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 -- 172.21.15.40:0/680100361 >> 172.21.15.40:0/680100361 conn(0x7fb1fc100910 msgr2=0x7fb1fc102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:45.228 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 -- 172.21.15.40:0/680100361 shutdown_connections 2023-07-14T11:02:45.228 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.093+0000 7fb2011e9700 1 -- 172.21.15.40:0/680100361 wait complete. 2023-07-14T11:02:45.228 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.094+0000 7fb2011e9700 1 Processor -- start 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.094+0000 7fb2011e9700 1 -- start start 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb2011e9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb2011e9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1fc19b660 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb1fad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb1fad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59938/0 (socket says 172.21.15.40:59938) 2023-07-14T11:02:45.229 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb1fad9d700 1 -- 172.21.15.40:0/1284013790 learned_addr learned my addr 172.21.15.40:0/1284013790 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:45.230 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.095+0000 7fb1fad9d700 1 -- 172.21.15.40:0/1284013790 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb1ec009550 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.230 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb1fad9d700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fb1ec006af0 tx=0x7fb1ec011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:45.230 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb1ec003a40 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.231 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb1ec003ba0 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.231 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb1ec01a700 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.231 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb1fc19b860 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.096+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb1fc19bc80 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.097+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7fb1ec024ad0 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.097+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb1fc061b40 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.098+0000 7fb1e3fff700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 0x7fb1e403ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.098+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fb1ec04c790 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.233 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.098+0000 7fb1fa59c700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 0x7fb1e403ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.098+0000 7fb1fa59c700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 0x7fb1e403ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fb1f0006fd0 tx=0x7fb1f0006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.102+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb1ec024430 con 0x7fb1fc07bdd0 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.311+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) v1 -- 0x7fb1fc101dd0 con 0x7fb1e4038760 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.720+0000 7fb1e3fff700 1 -- 172.21.15.40:0/1284013790 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7fb1fc101dd0 con 0x7fb1e4038760 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 msgr2=0x7fb1e403ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 0x7fb1e403ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fb1f0006fd0 tx=0x7fb1f0006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 msgr2=0x7fb1fc19b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:45.234 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fb1ec006af0 tx=0x7fb1ec011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:45.235 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 shutdown_connections 2023-07-14T11:02:45.235 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fb1e4038760 0x7fb1e403ac10 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:45.235 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 --2- 172.21.15.40:0/1284013790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1fc07bdd0 0x7fb1fc19b120 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:45.235 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.722+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 >> 172.21.15.40:0/1284013790 conn(0x7fb1fc100910 msgr2=0x7fb1fc1016c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:45.236 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.723+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 shutdown_connections 2023-07-14T11:02:45.236 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:44.723+0000 7fb2011e9700 1 -- 172.21.15.40:0/1284013790 wait complete. 2023-07-14T11:02:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:45 smithi040 ceph-mon[97954]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:45.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:45 smithi040 ceph-mon[97954]: Generating ssh key... 2023-07-14T11:02:45.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:45 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:45.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:45 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:46.510 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:02:46.510 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.792+0000 7f68829ed700 1 Processor -- start 2023-07-14T11:02:46.511 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.792+0000 7f68829ed700 1 -- start start 2023-07-14T11:02:46.511 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.792+0000 7f68829ed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:46.511 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.792+0000 7f68829ed700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f687c07a880 con 0x7f687c07bdf0 2023-07-14T11:02:46.511 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.793+0000 7f687bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:46.512 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.793+0000 7f687bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59954/0 (socket says 172.21.15.40:59954) 2023-07-14T11:02:46.512 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.793+0000 7f687bfff700 1 -- 172.21.15.40:0/1205285115 learned_addr learned my addr 172.21.15.40:0/1205285115 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:46.512 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.793+0000 7f687bfff700 1 -- 172.21.15.40:0/1205285115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f687c07a9c0 con 0x7f687c07bdf0 2023-07-14T11:02:46.512 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.793+0000 7f687bfff700 1 --2- 172.21.15.40:0/1205285115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f686c0098a0 tx=0x7f686c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b3fb124900f169e8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:46.512 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f687affd700 1 -- 172.21.15.40:0/1205285115 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f686c004160 con 0x7f687c07bdf0 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f687affd700 1 -- 172.21.15.40:0/1205285115 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f686c0042c0 con 0x7f687c07bdf0 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f687affd700 1 -- 172.21.15.40:0/1205285115 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f686c011a00 con 0x7f687c07bdf0 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 -- 172.21.15.40:0/1205285115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 msgr2=0x7f687c07a340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 --2- 172.21.15.40:0/1205285115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f686c0098a0 tx=0x7f686c009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 -- 172.21.15.40:0/1205285115 shutdown_connections 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 --2- 172.21.15.40:0/1205285115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c07a340 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 -- 172.21.15.40:0/1205285115 >> 172.21.15.40:0/1205285115 conn(0x7f687c100910 msgr2=0x7f687c102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:46.513 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.794+0000 7f68829ed700 1 -- 172.21.15.40:0/1205285115 shutdown_connections 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.795+0000 7f68829ed700 1 -- 172.21.15.40:0/1205285115 wait complete. 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.795+0000 7f68829ed700 1 Processor -- start 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f68829ed700 1 -- start start 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f68829ed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f68829ed700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f687c19b670 con 0x7f687c07bdf0 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f687bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:46.514 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f687bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59960/0 (socket says 172.21.15.40:59960) 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.796+0000 7f687bfff700 1 -- 172.21.15.40:0/1886405612 learned_addr learned my addr 172.21.15.40:0/1886405612 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f687bfff700 1 -- 172.21.15.40:0/1886405612 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f686c009550 con 0x7f687c07bdf0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f687bfff700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f686c006af0 tx=0x7f686c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f686c003a40 con 0x7f687c07bdf0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f686c003ba0 con 0x7f687c07bdf0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f686c01a700 con 0x7f687c07bdf0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f687c19b870 con 0x7f687c07bdf0 2023-07-14T11:02:46.515 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.797+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f687c19bc90 con 0x7f687c07bdf0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.798+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f686c024ad0 con 0x7f687c07bdf0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.798+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f687c10b550 con 0x7f687c07bdf0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.799+0000 7f68797fa700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 0x7f686403ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.799+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f686c04c770 con 0x7f687c07bdf0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.799+0000 7f687b7fe700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 0x7f686403ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.800+0000 7f687b7fe700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 0x7f686403ac10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6870006fd0 tx=0x7f6870006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:45.804+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f686c024430 con 0x7f687c07bdf0 2023-07-14T11:02:46.516 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.013+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) v1 -- 0x7f687c19eaf0 con 0x7f6864038760 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.014+0000 7f68797fa700 1 -- 172.21.15.40:0/1886405612 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+595 (secure 0 0 0) 0x7f687c19eaf0 con 0x7f6864038760 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.016+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 msgr2=0x7f686403ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.016+0000 7f68829ed700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 0x7f686403ac10 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6870006fd0 tx=0x7f6870006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.016+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 msgr2=0x7f687c19b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.016+0000 7f68829ed700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f686c006af0 tx=0x7f686c011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 shutdown_connections 2023-07-14T11:02:46.517 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f6864038760 0x7f686403ac10 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:46.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 --2- 172.21.15.40:0/1886405612 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f687c07bdf0 0x7f687c19b130 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:46.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 >> 172.21.15.40:0/1886405612 conn(0x7f687c100910 msgr2=0x7f687c1016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:46.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 shutdown_connections 2023-07-14T11:02:46.518 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:46.017+0000 7f68829ed700 1 -- 172.21.15.40:0/1886405612 wait complete. 2023-07-14T11:02:46.519 INFO:teuthology.orchestra.run.smithi040.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-07-14T11:02:46.519 INFO:teuthology.orchestra.run.smithi040.stdout:Adding key to root@localhost authorized_keys... 2023-07-14T11:02:46.519 INFO:teuthology.orchestra.run.smithi040.stdout:Adding host smithi040... 2023-07-14T11:02:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:46 smithi040 ceph-mon[97954]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:48.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:47 smithi040 ceph-mon[97954]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi040", "addr": "172.21.15.40", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:48 smithi040 ceph-mon[97954]: Deploying cephadm binary to smithi040 2023-07-14T11:02:51.001 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout Added host 'smithi040' with addr '172.21.15.40' 2023-07-14T11:02:51.001 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f610a68f700 1 Processor -- start 2023-07-14T11:02:51.001 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f610a68f700 1 -- start start 2023-07-14T11:02:51.001 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f610a68f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f610a68f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f610407c1a0 con 0x7f610407bd90 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f6103fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f6103fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59970/0 (socket says 172.21.15.40:59970) 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.078+0000 7f6103fff700 1 -- 172.21.15.40:0/3426739433 learned_addr learned my addr 172.21.15.40:0/3426739433 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.079+0000 7f6103fff700 1 -- 172.21.15.40:0/3426739433 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f610407a820 con 0x7f610407bd90 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.079+0000 7f6103fff700 1 --2- 172.21.15.40:0/3426739433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f60f40098a0 tx=0x7f60f4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7cf37e473802df95 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:51.002 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.079+0000 7f6102ffd700 1 -- 172.21.15.40:0/3426739433 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f60f4004160 con 0x7f610407bd90 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.079+0000 7f6102ffd700 1 -- 172.21.15.40:0/3426739433 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f60f40042c0 con 0x7f610407bd90 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.079+0000 7f6102ffd700 1 -- 172.21.15.40:0/3426739433 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f60f4011a00 con 0x7f610407bd90 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 -- 172.21.15.40:0/3426739433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 msgr2=0x7f610407a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 --2- 172.21.15.40:0/3426739433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f60f40098a0 tx=0x7f60f4009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 -- 172.21.15.40:0/3426739433 shutdown_connections 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 --2- 172.21.15.40:0/3426739433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610407a2e0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 -- 172.21.15.40:0/3426739433 >> 172.21.15.40:0/3426739433 conn(0x7f6104100910 msgr2=0x7f6104102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:51.003 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 -- 172.21.15.40:0/3426739433 shutdown_connections 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.080+0000 7f610a68f700 1 -- 172.21.15.40:0/3426739433 wait complete. 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.081+0000 7f610a68f700 1 Processor -- start 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.081+0000 7f610a68f700 1 -- start start 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f610a68f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f610a68f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f610419b660 con 0x7f610407bd90 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f6103fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f6103fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59986/0 (socket says 172.21.15.40:59986) 2023-07-14T11:02:51.004 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f6103fff700 1 -- 172.21.15.40:0/3949608624 learned_addr learned my addr 172.21.15.40:0/3949608624 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f6103fff700 1 -- 172.21.15.40:0/3949608624 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60f4009550 con 0x7f610407bd90 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.082+0000 7f6103fff700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f60f4009fb0 tx=0x7f60f4011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.083+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f60f4003b70 con 0x7f610407bd90 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.083+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f610419b860 con 0x7f610407bd90 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.083+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f60f4003cd0 con 0x7f610407bd90 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.083+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f60f40247a0 con 0x7f610407bd90 2023-07-14T11:02:51.005 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.083+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f610419bc80 con 0x7f610407bd90 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.084+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f60f401ab40 con 0x7f610407bd90 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.084+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6104061b40 con 0x7f610407bd90 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.084+0000 7f61017fa700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 0x7f60ec03ab70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.084+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f60f404c870 con 0x7f610407bd90 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.085+0000 7f61037fe700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 0x7f60ec03ab70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.085+0000 7f61037fe700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 0x7f60ec03ab70 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f60f8006fd0 tx=0x7f60f8006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.089+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f60f401a430 con 0x7f610407bd90 2023-07-14T11:02:51.006 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:47.300+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi040", "addr": "172.21.15.40", "target": ["mon-mgr", ""]}) v1 -- 0x7f6104101db0 con 0x7f60ec0386c0 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.557+0000 7f61017fa700 1 -- 172.21.15.40:0/3949608624 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+48 (secure 0 0 0) 0x7f6104101db0 con 0x7f60ec0386c0 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 msgr2=0x7f60ec03ab70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 0x7f60ec03ab70 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f60f8006fd0 tx=0x7f60f8006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 msgr2=0x7f610419b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f60f4009fb0 tx=0x7f60f4011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 shutdown_connections 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f60ec0386c0 0x7f60ec03ab70 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:51.007 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 --2- 172.21.15.40:0/3949608624 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f610407bd90 0x7f610419b120 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:51.008 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 >> 172.21.15.40:0/3949608624 conn(0x7f6104100910 msgr2=0x7f61041016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:51.008 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 shutdown_connections 2023-07-14T11:02:51.008 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:50.561+0000 7f610a68f700 1 -- 172.21.15.40:0/3949608624 wait complete. 2023-07-14T11:02:51.008 INFO:teuthology.orchestra.run.smithi040.stdout:Deploying unmanaged mon service... 2023-07-14T11:02:51.857 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:51 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:51.858 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:51 smithi040 ceph-mon[97954]: Added host smithi040 2023-07-14T11:02:51.858 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:51 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:02:52.387 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-07-14T11:02:52.387 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.588+0000 7f29b22bf700 1 Processor -- start 2023-07-14T11:02:52.387 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.588+0000 7f29b22bf700 1 -- start start 2023-07-14T11:02:52.387 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.588+0000 7f29b22bf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.588+0000 7f29b22bf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29ac07c1c0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29abfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29abfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59996/0 (socket says 172.21.15.40:59996) 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29abfff700 1 -- 172.21.15.40:0/256199608 learned_addr learned my addr 172.21.15.40:0/256199608 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29abfff700 1 -- 172.21.15.40:0/256199608 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29ac07a840 con 0x7f29ac07bdb0 2023-07-14T11:02:52.388 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29abfff700 1 --2- 172.21.15.40:0/256199608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f299c0098a0 tx=0x7f299c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3a2a99dc9d0d4fd9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29aaffd700 1 -- 172.21.15.40:0/256199608 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f299c004160 con 0x7f29ac07bdb0 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29aaffd700 1 -- 172.21.15.40:0/256199608 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f299c0042c0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.589+0000 7f29aaffd700 1 -- 172.21.15.40:0/256199608 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f299c011a00 con 0x7f29ac07bdb0 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 -- 172.21.15.40:0/256199608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 msgr2=0x7f29ac07a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 --2- 172.21.15.40:0/256199608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f299c0098a0 tx=0x7f299c009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 -- 172.21.15.40:0/256199608 shutdown_connections 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 --2- 172.21.15.40:0/256199608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac07a300 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:52.389 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 -- 172.21.15.40:0/256199608 >> 172.21.15.40:0/256199608 conn(0x7f29ac100910 msgr2=0x7f29ac102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 -- 172.21.15.40:0/256199608 shutdown_connections 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.590+0000 7f29b22bf700 1 -- 172.21.15.40:0/256199608 wait complete. 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.591+0000 7f29b22bf700 1 Processor -- start 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.591+0000 7f29b22bf700 1 -- start start 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29b22bf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29b22bf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29ac19b6a0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.390 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29abfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:52.391 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29abfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60006/0 (socket says 172.21.15.40:60006) 2023-07-14T11:02:52.391 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29abfff700 1 -- 172.21.15.40:0/3277515373 learned_addr learned my addr 172.21.15.40:0/3277515373 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:52.391 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.592+0000 7f29abfff700 1 -- 172.21.15.40:0/3277515373 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f299c009550 con 0x7f29ac07bdb0 2023-07-14T11:02:52.391 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29abfff700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f299c006af0 tx=0x7f299c011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:52.391 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f299c003a40 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f299c003ba0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f299c01a700 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29ac19b8a0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.593+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29ac19bcc0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.594+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f299c024ad0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.392 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.594+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29ac10b580 con 0x7f29ac07bdb0 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.595+0000 7f29a97fa700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 0x7f299403ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.595+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f299c04c6b0 con 0x7f29ac07bdb0 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.595+0000 7f29ab7fe700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 0x7f299403ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.596+0000 7f29ab7fe700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 0x7f299403ac10 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f29a0006fd0 tx=0x7f29a0006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.600+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f299c024430 con 0x7f29ac07bdb0 2023-07-14T11:02:52.393 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.824+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f29ac19eb20 con 0x7f2994038760 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.830+0000 7f29a97fa700 1 -- 172.21.15.40:0/3277515373 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f29ac19eb20 con 0x7f2994038760 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 msgr2=0x7f299403ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 0x7f299403ac10 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f29a0006fd0 tx=0x7f29a0006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 msgr2=0x7f29ac19b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f299c006af0 tx=0x7f299c011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 shutdown_connections 2023-07-14T11:02:52.394 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f2994038760 0x7f299403ac10 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:52.395 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 --2- 172.21.15.40:0/3277515373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29ac07bdb0 0x7f29ac19b160 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:52.395 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 >> 172.21.15.40:0/3277515373 conn(0x7f29ac100910 msgr2=0x7f29ac101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:52.395 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.832+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 shutdown_connections 2023-07-14T11:02:52.395 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:51.833+0000 7f29b22bf700 1 -- 172.21.15.40:0/3277515373 wait complete. 2023-07-14T11:02:52.395 INFO:teuthology.orchestra.run.smithi040.stdout:Deploying unmanaged mgr service... 2023-07-14T11:02:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:52 smithi040 ceph-mon[97954]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:52 smithi040 ceph-mon[97954]: Saving service mon spec with placement count:5 2023-07-14T11:02:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:52 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:53.767 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-07-14T11:02:53.767 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.058+0000 7f13e3de2700 1 Processor -- start 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.058+0000 7f13e3de2700 1 -- start start 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.058+0000 7f13e3de2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.058+0000 7f13e3de2700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc07c1a0 con 0x7f13dc07bd90 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.059+0000 7f13e1b7e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.059+0000 7f13e1b7e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60016/0 (socket says 172.21.15.40:60016) 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.059+0000 7f13e1b7e700 1 -- 172.21.15.40:0/2338344730 learned_addr learned my addr 172.21.15.40:0/2338344730 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:53.768 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.059+0000 7f13e1b7e700 1 -- 172.21.15.40:0/2338344730 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13dc07a820 con 0x7f13dc07bd90 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.059+0000 7f13e1b7e700 1 --2- 172.21.15.40:0/2338344730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f13d80098a0 tx=0x7f13d8009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6f8f005e6c8bc8a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e0b7c700 1 -- 172.21.15.40:0/2338344730 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d8004160 con 0x7f13dc07bd90 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e0b7c700 1 -- 172.21.15.40:0/2338344730 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f13d80042c0 con 0x7f13dc07bd90 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e0b7c700 1 -- 172.21.15.40:0/2338344730 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d8011a00 con 0x7f13dc07bd90 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 -- 172.21.15.40:0/2338344730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 msgr2=0x7f13dc07a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 --2- 172.21.15.40:0/2338344730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f13d80098a0 tx=0x7f13d8009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 -- 172.21.15.40:0/2338344730 shutdown_connections 2023-07-14T11:02:53.769 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 --2- 172.21.15.40:0/2338344730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc07a2e0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 -- 172.21.15.40:0/2338344730 >> 172.21.15.40:0/2338344730 conn(0x7f13dc100910 msgr2=0x7f13dc102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.060+0000 7f13e3de2700 1 -- 172.21.15.40:0/2338344730 shutdown_connections 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.061+0000 7f13e3de2700 1 -- 172.21.15.40:0/2338344730 wait complete. 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.061+0000 7f13e3de2700 1 Processor -- start 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e3de2700 1 -- start start 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e3de2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:53.770 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e3de2700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13dc19b660 con 0x7f13dc07bd90 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e1b7e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e1b7e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60022/0 (socket says 172.21.15.40:60022) 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.062+0000 7f13e1b7e700 1 -- 172.21.15.40:0/1676386193 learned_addr learned my addr 172.21.15.40:0/1676386193 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13e1b7e700 1 -- 172.21.15.40:0/1676386193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13d8009550 con 0x7f13dc07bd90 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13e1b7e700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f13d8006b20 tx=0x7f13d8011ad0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d8003b70 con 0x7f13dc07bd90 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f13d8003cd0 con 0x7f13dc07bd90 2023-07-14T11:02:53.771 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f13d80247c0 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13dc19b860 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.063+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13dc19bc80 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.064+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f13d801ab40 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.064+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13dc061b40 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.065+0000 7f13d2ffd700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 0x7f13c803ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.065+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f13d804c9d0 con 0x7f13dc07bd90 2023-07-14T11:02:53.772 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.065+0000 7f13e137d700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 0x7f13c803ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:53.773 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.066+0000 7f13e137d700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 0x7f13c803ac10 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f13cc006fd0 tx=0x7f13cc006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:53.773 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.070+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f13d801a950 con 0x7f13dc07bd90 2023-07-14T11:02:53.773 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.298+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f13dc101db0 con 0x7f13c8038760 2023-07-14T11:02:53.773 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.305+0000 7f13d2ffd700 1 -- 172.21.15.40:0/1676386193 <== mgr.14116 v2:172.21.15.40:6800/2376955220 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f13dc101db0 con 0x7f13c8038760 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 msgr2=0x7f13c803ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 0x7f13c803ac10 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f13cc006fd0 tx=0x7f13cc006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 msgr2=0x7f13dc19b120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f13d8006b20 tx=0x7f13d8011ad0 comp rx=0 tx=0).stop 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 shutdown_connections 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f13c8038760 0x7f13c803ac10 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 --2- 172.21.15.40:0/1676386193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13dc07bd90 0x7f13dc19b120 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:53.774 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 >> 172.21.15.40:0/1676386193 conn(0x7f13dc100910 msgr2=0x7f13dc1016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:53.775 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 shutdown_connections 2023-07-14T11:02:53.775 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:53.308+0000 7f13e3de2700 1 -- 172.21.15.40:0/1676386193 wait complete. 2023-07-14T11:02:54.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:54 smithi040 ceph-mon[97954]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:02:54.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:54 smithi040 ceph-mon[97954]: Saving service mgr spec with placement count:2 2023-07-14T11:02:54.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:54 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:54.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:54 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:54.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:54 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:54.942 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.299+0000 7f4b34f6e700 1 Processor -- start 2023-07-14T11:02:54.942 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.299+0000 7f4b34f6e700 1 -- start start 2023-07-14T11:02:54.942 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.300+0000 7f4b34f6e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:54.942 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.300+0000 7f4b34f6e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b3007c1c0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.943 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.300+0000 7f4b2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:54.943 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.300+0000 7f4b2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56866/0 (socket says 172.21.15.40:56866) 2023-07-14T11:02:54.943 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.300+0000 7f4b2e59c700 1 -- 172.21.15.40:0/2154447823 learned_addr learned my addr 172.21.15.40:0/2154447823 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:54.947 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.301+0000 7f4b2e59c700 1 -- 172.21.15.40:0/2154447823 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b3007a840 con 0x7f4b3007bdb0 2023-07-14T11:02:54.947 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.301+0000 7f4b2e59c700 1 --2- 172.21.15.40:0/2154447823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f4b200098a0 tx=0x7f4b20009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e65ad79292960645 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:54.947 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.301+0000 7f4b2d59a700 1 -- 172.21.15.40:0/2154447823 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4b20004160 con 0x7f4b3007bdb0 2023-07-14T11:02:54.947 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.301+0000 7f4b2d59a700 1 -- 172.21.15.40:0/2154447823 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b200042c0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.301+0000 7f4b2d59a700 1 -- 172.21.15.40:0/2154447823 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4b200119b0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2154447823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 msgr2=0x7f4b3007a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2154447823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f4b200098a0 tx=0x7f4b20009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2154447823 shutdown_connections 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2154447823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3007a300 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:54.948 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2154447823 >> 172.21.15.40:0/2154447823 conn(0x7f4b30100890 msgr2=0x7f4b30102cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2154447823 shutdown_connections 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.302+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2154447823 wait complete. 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.303+0000 7f4b34f6e700 1 Processor -- start 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.303+0000 7f4b34f6e700 1 -- start start 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b34f6e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b34f6e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b3019b6c0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.949 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56870/0 (socket says 172.21.15.40:56870) 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b2e59c700 1 -- 172.21.15.40:0/2035669594 learned_addr learned my addr 172.21.15.40:0/2035669594 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b2e59c700 1 -- 172.21.15.40:0/2035669594 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b20009550 con 0x7f4b3007bdb0 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.304+0000 7f4b2e59c700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f4b20003e60 tx=0x7f4b20011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.305+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4b20003a60 con 0x7f4b3007bdb0 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.305+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b20003bc0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.950 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.305+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b3019b8c0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.305+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f4b200196c0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.305+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b3019bce0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.306+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f4b2001a430 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.307+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b30061b40 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.307+0000 7f4b1f7fe700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 0x7f4b1803aba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.307+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f4b2004c9e0 con 0x7f4b3007bdb0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.307+0000 7f4b2dd9b700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 0x7f4b1803aba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:54.951 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.308+0000 7f4b2dd9b700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 0x7f4b1803aba0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f4b24006fd0 tx=0x7f4b24006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.312+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4b20010560 con 0x7f4b3007bdb0 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.521+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) v1 -- 0x7f4b3019eb40 con 0x7f4b3007bdb0 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.528+0000 7f4b1f7fe700 1 -- 172.21.15.40:0/2035669594 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) v1 ==== 142+0+0 (secure 0 0 0) 0x7f4b20010380 con 0x7f4b3007bdb0 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.530+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 msgr2=0x7f4b1803aba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.530+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 0x7f4b1803aba0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f4b24006fd0 tx=0x7f4b24006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.530+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 msgr2=0x7f4b3019b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:54.952 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.530+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f4b20003e60 tx=0x7f4b20011a30 comp rx=0 tx=0).stop 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.530+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 shutdown_connections 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.531+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f4b180386f0 0x7f4b1803aba0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.531+0000 7f4b34f6e700 1 --2- 172.21.15.40:0/2035669594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b3007bdb0 0x7f4b3019b180 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.531+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 >> 172.21.15.40:0/2035669594 conn(0x7f4b30100890 msgr2=0x7f4b30101f30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.531+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 shutdown_connections 2023-07-14T11:02:54.953 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:54.531+0000 7f4b34f6e700 1 -- 172.21.15.40:0/2035669594 wait complete. 2023-07-14T11:02:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:55 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2035669594' entity='client.admin' 2023-07-14T11:02:56.326 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.565+0000 7fd273c6f700 1 Processor -- start 2023-07-14T11:02:56.327 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.565+0000 7fd273c6f700 1 -- start start 2023-07-14T11:02:56.327 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.565+0000 7fd273c6f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:56.327 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.565+0000 7fd273c6f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd26c07c1c0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.328 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.565+0000 7fd271a0b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:56.328 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd271a0b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56876/0 (socket says 172.21.15.40:56876) 2023-07-14T11:02:56.328 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd271a0b700 1 -- 172.21.15.40:0/3601546959 learned_addr learned my addr 172.21.15.40:0/3601546959 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:56.328 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd271a0b700 1 -- 172.21.15.40:0/3601546959 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd26c07a840 con 0x7fd26c07bdb0 2023-07-14T11:02:56.329 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd271a0b700 1 --2- 172.21.15.40:0/3601546959 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd2680098a0 tx=0x7fd268009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b7cda72b5e00a08 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:56.329 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd270a09700 1 -- 172.21.15.40:0/3601546959 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd268004160 con 0x7fd26c07bdb0 2023-07-14T11:02:56.329 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd270a09700 1 -- 172.21.15.40:0/3601546959 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd2680042c0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.329 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.566+0000 7fd270a09700 1 -- 172.21.15.40:0/3601546959 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd2680119b0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.330 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 -- 172.21.15.40:0/3601546959 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 msgr2=0x7fd26c07a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:56.330 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3601546959 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd2680098a0 tx=0x7fd268009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:56.330 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 -- 172.21.15.40:0/3601546959 shutdown_connections 2023-07-14T11:02:56.331 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3601546959 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c07a300 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:56.331 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 -- 172.21.15.40:0/3601546959 >> 172.21.15.40:0/3601546959 conn(0x7fd26c100910 msgr2=0x7fd26c102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:56.331 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 -- 172.21.15.40:0/3601546959 shutdown_connections 2023-07-14T11:02:56.331 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.567+0000 7fd273c6f700 1 -- 172.21.15.40:0/3601546959 wait complete. 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.568+0000 7fd273c6f700 1 Processor -- start 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.568+0000 7fd273c6f700 1 -- start start 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd273c6f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd273c6f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd26c19b6c0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd271a0b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:56.332 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd271a0b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56882/0 (socket says 172.21.15.40:56882) 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd271a0b700 1 -- 172.21.15.40:0/3790567465 learned_addr learned my addr 172.21.15.40:0/3790567465 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd271a0b700 1 -- 172.21.15.40:0/3790567465 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd268009550 con 0x7fd26c07bdb0 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.569+0000 7fd271a0b700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fd268000c00 tx=0x7fd268011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.570+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd2680039f0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.570+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd26c19b8c0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.333 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.570+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd268003b50 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.570+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fd26801a780 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.570+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd26c19bce0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.571+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd26c061b40 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.571+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7fd268024440 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.572+0000 7fd262ffd700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 0x7fd25803abc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.572+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7fd26804c850 con 0x7fd26c07bdb0 2023-07-14T11:02:56.334 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.572+0000 7fd27120a700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 0x7fd25803abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.573+0000 7fd27120a700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 0x7fd25803abc0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fd25c006fd0 tx=0x7fd25c006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.576+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fd2680189a0 con 0x7fd26c07bdb0 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.790+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) v1 -- 0x7fd26c19eb40 con 0x7fd26c07bdb0 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.796+0000 7fd262ffd700 1 -- 172.21.15.40:0/3790567465 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) v1 ==== 130+0+0 (secure 0 0 0) 0x7fd268018350 con 0x7fd26c07bdb0 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 msgr2=0x7fd25803abc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 0x7fd25803abc0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fd25c006fd0 tx=0x7fd25c006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:56.335 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 msgr2=0x7fd26c19b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fd268000c00 tx=0x7fd268011a30 comp rx=0 tx=0).stop 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 shutdown_connections 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7fd258038710 0x7fd25803abc0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 --2- 172.21.15.40:0/3790567465 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd26c07bdb0 0x7fd26c19b180 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.799+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 >> 172.21.15.40:0/3790567465 conn(0x7fd26c100910 msgr2=0x7fd26c101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:56.336 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.800+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 shutdown_connections 2023-07-14T11:02:56.337 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:55.800+0000 7fd273c6f700 1 -- 172.21.15.40:0/3790567465 wait complete. 2023-07-14T11:02:56.337 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling the dashboard module... 2023-07-14T11:02:56.910 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:56 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3790567465' entity='client.admin' 2023-07-14T11:02:56.910 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:56 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:58.091 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:57 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4203688739' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-07-14T11:02:58.092 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:57 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setuser ceph since I am not root 2023-07-14T11:02:58.092 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:57 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setgroup ceph since I am not root 2023-07-14T11:02:58.411 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:58.106+0000 7f17b41bf200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:02:58.411 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:58.213+0000 7f17b41bf200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:02:58.412 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:58.331+0000 7f17b41bf200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:02:58.416 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.888+0000 7f7eb8a31700 1 Processor -- start 2023-07-14T11:02:58.416 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.888+0000 7f7eb8a31700 1 -- start start 2023-07-14T11:02:58.416 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb8a31700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:58.416 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb8a31700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb407a880 con 0x7f7eb407bdf0 2023-07-14T11:02:58.417 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:58.417 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56896/0 (socket says 172.21.15.40:56896) 2023-07-14T11:02:58.417 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb259c700 1 -- 172.21.15.40:0/3929656580 learned_addr learned my addr 172.21.15.40:0/3929656580 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:58.418 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.889+0000 7f7eb259c700 1 -- 172.21.15.40:0/3929656580 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7eb407a9c0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.418 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.890+0000 7f7eb259c700 1 --2- 172.21.15.40:0/3929656580 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f7ea40098a0 tx=0x7f7ea4009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9df531ac21846c7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:58.418 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.890+0000 7f7eb159a700 1 -- 172.21.15.40:0/3929656580 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7ea4004160 con 0x7f7eb407bdf0 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.890+0000 7f7eb159a700 1 -- 172.21.15.40:0/3929656580 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ea40042c0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.890+0000 7f7eb159a700 1 -- 172.21.15.40:0/3929656580 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7ea40119b0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 -- 172.21.15.40:0/3929656580 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 msgr2=0x7f7eb407a340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/3929656580 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f7ea40098a0 tx=0x7f7ea4009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 -- 172.21.15.40:0/3929656580 shutdown_connections 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/3929656580 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb407a340 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:58.419 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 -- 172.21.15.40:0/3929656580 >> 172.21.15.40:0/3929656580 conn(0x7f7eb4100910 msgr2=0x7f7eb4102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 -- 172.21.15.40:0/3929656580 shutdown_connections 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.891+0000 7f7eb8a31700 1 -- 172.21.15.40:0/3929656580 wait complete. 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.892+0000 7f7eb8a31700 1 Processor -- start 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.892+0000 7f7eb8a31700 1 -- start start 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.892+0000 7f7eb8a31700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.892+0000 7f7eb8a31700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb419b680 con 0x7f7eb407bdf0 2023-07-14T11:02:58.420 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7eb259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7eb259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56912/0 (socket says 172.21.15.40:56912) 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7eb259c700 1 -- 172.21.15.40:0/4203688739 learned_addr learned my addr 172.21.15.40:0/4203688739 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7eb259c700 1 -- 172.21.15.40:0/4203688739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea4009550 con 0x7f7eb407bdf0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7eb259c700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4000c00 tx=0x7f7ea4011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.893+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7ea40039f0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.894+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ea4003b50 con 0x7f7eb407bdf0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.894+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7eb419b880 con 0x7f7eb407bdf0 2023-07-14T11:02:58.421 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.894+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f7ea401a780 con 0x7f7eb407bdf0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.894+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7eb419bca0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.895+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7eb410b5a0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.896+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 45483+0+0 (secure 0 0 0) 0x7f7ea401a9f0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.896+0000 7f7ea37fe700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 0x7f7e9c03ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.896+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f7ea404cb20 con 0x7f7eb407bdf0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.897+0000 7f7eb1d9b700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 0x7f7e9c03ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:58.422 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.897+0000 7f7eb1d9b700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 0x7f7e9c03ac10 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f7ea8006fd0 tx=0x7f7ea8006e40 comp rx=0 tx=0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:56.901+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f7ea4018940 con 0x7f7eb407bdf0 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.169+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) v1 -- 0x7f7eb410b790 con 0x7f7eb407bdf0 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.805+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 9) v1 ==== 45496+0+0 (secure 0 0 0) 0x7f7ea40193c0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.807+0000 7f7ea37fe700 1 -- 172.21.15.40:0/4203688739 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) v1 ==== 88+0+0 (secure 0 0 0) 0x7f7ea400b7e0 con 0x7f7eb407bdf0 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 msgr2=0x7f7e9c03ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 0x7f7e9c03ac10 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f7ea8006fd0 tx=0x7f7ea8006e40 comp rx=0 tx=0).stop 2023-07-14T11:02:58.423 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 msgr2=0x7f7eb419b140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4000c00 tx=0x7f7ea4011a30 comp rx=0 tx=0).stop 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 shutdown_connections 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f7e9c038760 0x7f7e9c03ac10 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 --2- 172.21.15.40:0/4203688739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb407bdf0 0x7f7eb419b140 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 >> 172.21.15.40:0/4203688739 conn(0x7f7eb4100910 msgr2=0x7f7eb41016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 shutdown_connections 2023-07-14T11:02:58.424 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:57.810+0000 7f7eb8a31700 1 -- 172.21.15.40:0/4203688739 wait complete. 2023-07-14T11:02:58.688 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:58.470+0000 7f17b41bf200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:02:59.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:58.708+0000 7f17b41bf200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:02:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4203688739' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-07-14T11:02:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-mon[97954]: mgrmap e9: x(active, since 18s) 2023-07-14T11:02:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:02:58 smithi040 ceph-mon[97954]: from='mgr.14116 172.21.15.40:0/566449584' entity='mgr.x' 2023-07-14T11:02:59.399 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:59.118+0000 7f17b41bf200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:02:59.399 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:59.237+0000 7f17b41bf200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:02:59.667 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:59.417+0000 7f17b41bf200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:02:59.667 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:02:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:02:59.545+0000 7f17b41bf200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:02:59.711 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:02:59.715 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-07-14T11:02:59.715 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "available": true, 2023-07-14T11:02:59.716 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "active_name": "x", 2023-07-14T11:02:59.716 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.966+0000 7f0cc85df700 1 Processor -- start 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.966+0000 7f0cc85df700 1 -- start start 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.966+0000 7f0cc85df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.966+0000 7f0cc85df700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cc007a880 con 0x7f0cc007bdf0 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.967+0000 7f0cc637b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.967+0000 7f0cc637b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56938/0 (socket says 172.21.15.40:56938) 2023-07-14T11:02:59.717 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.967+0000 7f0cc637b700 1 -- 172.21.15.40:0/2742177485 learned_addr learned my addr 172.21.15.40:0/2742177485 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.967+0000 7f0cc637b700 1 -- 172.21.15.40:0/2742177485 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cc007a9c0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.967+0000 7f0cc637b700 1 --2- 172.21.15.40:0/2742177485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f0cbc0098a0 tx=0x7f0cbc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c39d6bd619b30700 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc5379700 1 -- 172.21.15.40:0/2742177485 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0cbc004160 con 0x7f0cc007bdf0 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc5379700 1 -- 172.21.15.40:0/2742177485 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0cbc0042c0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc5379700 1 -- 172.21.15.40:0/2742177485 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0cbc0119b0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc85df700 1 -- 172.21.15.40:0/2742177485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 msgr2=0x7f0cc007a340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:59.718 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2742177485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f0cbc0098a0 tx=0x7f0cbc009bb0 comp rx=0 tx=0).stop 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc85df700 1 -- 172.21.15.40:0/2742177485 shutdown_connections 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2742177485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc007a340 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.968+0000 7f0cc85df700 1 -- 172.21.15.40:0/2742177485 >> 172.21.15.40:0/2742177485 conn(0x7f0cc0100910 msgr2=0x7f0cc0102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.969+0000 7f0cc85df700 1 -- 172.21.15.40:0/2742177485 shutdown_connections 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.969+0000 7f0cc85df700 1 -- 172.21.15.40:0/2742177485 wait complete. 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc85df700 1 Processor -- start 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc85df700 1 -- start start 2023-07-14T11:02:59.719 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc85df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc85df700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0cc019b680 con 0x7f0cc007bdf0 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc637b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc637b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56942/0 (socket says 172.21.15.40:56942) 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.970+0000 7f0cc637b700 1 -- 172.21.15.40:0/2200200770 learned_addr learned my addr 172.21.15.40:0/2200200770 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cc637b700 1 -- 172.21.15.40:0/2200200770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0cbc009550 con 0x7f0cc007bdf0 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cc637b700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f0cbc000c00 tx=0x7f0cbc011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:02:59.720 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0cbc0039f0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0cbc003b50 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0cc019b880 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f0cbc01a780 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.971+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0cc019bca0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.972+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 9) v1 ==== 45496+0+0 (secure 0 0 0) 0x7f0cbc024440 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.973+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0cc0061b40 con 0x7f0cc007bdf0 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.973+0000 7f0cb77fe700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 0x7f0cac03ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:02:59.721 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.973+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f0cbc04c880 con 0x7f0cc007bdf0 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.973+0000 7f0cc5b7a700 1 -- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 msgr2=0x7f0cac03ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.973+0000 7f0cc5b7a700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 0x7f0cac03ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:58.978+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0cbc0189a0 con 0x7f0cc007bdf0 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.174+0000 7f0cc5b7a700 1 -- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 msgr2=0x7f0cac03ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.174+0000 7f0cc5b7a700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 0x7f0cac03ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.247+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f0cc019ed90 con 0x7f0cc007bdf0 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.247+0000 7f0cb77fe700 1 -- 172.21.15.40:0/2200200770 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) v1 ==== 56+0+88 (secure 0 0 0) 0x7f0cbc018350 con 0x7f0cc007bdf0 2023-07-14T11:02:59.722 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 msgr2=0x7f0cac03ac60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 0x7f0cac03ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 msgr2=0x7f0cc019b140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f0cbc000c00 tx=0x7f0cbc011a30 comp rx=0 tx=0).stop 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 shutdown_connections 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f0cac0387b0 0x7f0cac03ac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 --2- 172.21.15.40:0/2200200770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0cc007bdf0 0x7f0cc019b140 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:02:59.723 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 >> 172.21.15.40:0/2200200770 conn(0x7f0cc0100910 msgr2=0x7f0cc01016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:02:59.724 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 shutdown_connections 2023-07-14T11:02:59.724 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:02:59.250+0000 7f0cc85df700 1 -- 172.21.15.40:0/2200200770 wait complete. 2023-07-14T11:02:59.724 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for the mgr to restart... 2023-07-14T11:02:59.724 INFO:teuthology.orchestra.run.smithi040.stdout:Waiting for mgr epoch 9... 2023-07-14T11:03:00.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:00 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2200200770' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-07-14T11:03:00.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:00.120+0000 7f17b41bf200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:03:01.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:00.892+0000 7f17b41bf200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:03:01.328 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:01.103+0000 7f17b41bf200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:03:02.432 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:02.077+0000 7f17b41bf200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:03:02.433 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:02.201+0000 7f17b41bf200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:03:02.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:02.428+0000 7f17b41bf200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:03:03.082 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:02.799+0000 7f17b41bf200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:03:03.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:03.079+0000 7f17b41bf200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:03:03.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:03.276+0000 7f17b41bf200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:03:04.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:03.628+0000 7f17b41bf200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:03:05.014 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:04.580+0000 7f17b41bf200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:03:05.014 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:04.758+0000 7f17b41bf200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:03:05.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:05.010+0000 7f17b41bf200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:03:05.297 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:05.162+0000 7f17b41bf200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:03:05.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:05.446+0000 7f17b41bf200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:03:06.796 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:03:06.538+0000 7f17b41bf200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: Active manager daemon x restarted 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: Activating manager daemon x 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: osdmap e3: 0 total, 0 up, 0 in 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: mgrmap e10: x(active, starting, since 0.00424569s) 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:03:06.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: Manager daemon x is now available 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:06.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:08.017 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout { 2023-07-14T11:03:08.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-07-14T11:03:08.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout "initialized": true 2023-07-14T11:03:08.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout } 2023-07-14T11:03:08.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.292+0000 7f5a5bb2b700 1 Processor -- start 2023-07-14T11:03:08.018 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.293+0000 7f5a5bb2b700 1 -- start start 2023-07-14T11:03:08.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.293+0000 7f5a5bb2b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:08.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.293+0000 7f5a5bb2b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a5407c1a0 con 0x7f5a5407bd90 2023-07-14T11:03:08.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a598c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:08.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a598c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56950/0 (socket says 172.21.15.40:56950) 2023-07-14T11:03:08.019 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a598c7700 1 -- 172.21.15.40:0/2216453951 learned_addr learned my addr 172.21.15.40:0/2216453951 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a598c7700 1 -- 172.21.15.40:0/2216453951 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a5407a820 con 0x7f5a5407bd90 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a598c7700 1 --2- 172.21.15.40:0/2216453951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f5a500098a0 tx=0x7f5a50009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=36294b46c975d261 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.294+0000 7f5a588c5700 1 -- 172.21.15.40:0/2216453951 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5a50004160 con 0x7f5a5407bd90 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a588c5700 1 -- 172.21.15.40:0/2216453951 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5a500042c0 con 0x7f5a5407bd90 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a588c5700 1 -- 172.21.15.40:0/2216453951 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5a500119b0 con 0x7f5a5407bd90 2023-07-14T11:03:08.020 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/2216453951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 msgr2=0x7f5a5407a2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:08.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/2216453951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f5a500098a0 tx=0x7f5a50009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/2216453951 shutdown_connections 2023-07-14T11:03:08.021 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/2216453951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5407a2e0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/2216453951 >> 172.21.15.40:0/2216453951 conn(0x7f5a54100910 msgr2=0x7f5a54102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:08.022 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/2216453951 shutdown_connections 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.295+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/2216453951 wait complete. 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.296+0000 7f5a5bb2b700 1 Processor -- start 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a5bb2b700 1 -- start start 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a5bb2b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a5bb2b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5a5419b650 con 0x7f5a5407bd90 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a598c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:08.023 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a598c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56954/0 (socket says 172.21.15.40:56954) 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.297+0000 7f5a598c7700 1 -- 172.21.15.40:0/1044905707 learned_addr learned my addr 172.21.15.40:0/1044905707 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a598c7700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5a50009550 con 0x7f5a5407bd90 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a598c7700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f5a50000c00 tx=0x7f5a50011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5a500039f0 con 0x7f5a5407bd90 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5a50003b50 con 0x7f5a5407bd90 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5a5419b850 con 0x7f5a5407bd90 2023-07-14T11:03:08.024 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5a5001a780 con 0x7f5a5407bd90 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.298+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5a5419bc70 con 0x7f5a5407bd90 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.300+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 9) v1 ==== 45496+0+0 (secure 0 0 0) 0x7f5a50024440 con 0x7f5a5407bd90 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.300+0000 7f5a4affd700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.300+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f5a4003b370 con 0x7f5a400387b0 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.300+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 940+0+0 (secure 0 0 0) 0x7f5a5004c910 con 0x7f5a5407bd90 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.301+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.025 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.301+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.502+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.502+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.903+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:00.903+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:01.704+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:01.704+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:03.305+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.026 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:03.306+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:06.509+0000 7f5a590c6700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2376955220 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:06.509+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 6.400000 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:06.543+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mgrmap(e 10) v1 ==== 45263+0+0 (secure 0 0 0) 0x7f5a5002d910 con 0x7f5a5407bd90 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:06.543+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 msgr2=0x7f5a4003ac60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:06.543+0000 7f5a4affd700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.545+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 11) v1 ==== 45390+0+0 (secure 0 0 0) 0x7f5a5004e230 con 0x7f5a5407bd90 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.545+0000 7f5a4affd700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 0x7f5a4003e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:08.027 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.545+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f5a4003b370 con 0x7f5a4003bff0 2023-07-14T11:03:08.028 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.546+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 0x7f5a4003e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:08.028 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.548+0000 7f5a590c6700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 0x7f5a4003e3d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f5a44003a10 tx=0x7f5a44009270 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:08.028 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.549+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6910 (secure 0 0 0) 0x7f5a4003b370 con 0x7f5a4003bff0 2023-07-14T11:03:08.028 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f5a5419c450 con 0x7f5a4003bff0 2023-07-14T11:03:08.028 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a4affd700 1 -- 172.21.15.40:0/1044905707 <== mgr.14148 v2:172.21.15.40:6800/2087222828 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+52 (secure 0 0 0) 0x7f5a5419c450 con 0x7f5a4003bff0 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 msgr2=0x7f5a4003e3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 0x7f5a4003e3d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f5a44003a10 tx=0x7f5a44009270 comp rx=0 tx=0).stop 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 msgr2=0x7f5a5419b110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.556+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f5a50000c00 tx=0x7f5a50011a30 comp rx=0 tx=0).stop 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 shutdown_connections 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5a4003bff0 0x7f5a4003e3d0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.029 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5a5407bd90 0x7f5a5419b110 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.030 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 --2- 172.21.15.40:0/1044905707 >> [v2:172.21.15.40:6800/2376955220,v1:172.21.15.40:6801/2376955220] conn(0x7f5a400387b0 0x7f5a4003ac60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:08.030 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 >> 172.21.15.40:0/1044905707 conn(0x7f5a54100910 msgr2=0x7f5a541016a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:08.030 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 shutdown_connections 2023-07-14T11:03:08.031 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:07.557+0000 7f5a5bb2b700 1 -- 172.21.15.40:0/1044905707 wait complete. 2023-07-14T11:03:08.031 INFO:teuthology.orchestra.run.smithi040.stdout:mgr epoch 9 is available 2023-07-14T11:03:08.031 INFO:teuthology.orchestra.run.smithi040.stdout:Generating a dashboard self-signed certificate... 2023-07-14T11:03:08.032 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:03:08.035 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:03:08.035 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:07 smithi040 ceph-mon[97954]: mgrmap e11: x(active, since 1.00809s) 2023-07-14T11:03:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: [14/Jul/2023:11:03:07] ENGINE Bus STARTING 2023-07-14T11:03:08.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: [14/Jul/2023:11:03:07] ENGINE Serving on http://172.21.15.40:8765 2023-07-14T11:03:08.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: [14/Jul/2023:11:03:07] ENGINE Serving on https://172.21.15.40:7150 2023-07-14T11:03:08.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: [14/Jul/2023:11:03:07] ENGINE Bus STARTED 2023-07-14T11:03:08.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-07-14T11:03:08.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:08 smithi040 ceph-mon[97954]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-07-14T11:03:09.494 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.760+0000 7fb3bbd49700 1 Processor -- start 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.760+0000 7fb3bbd49700 1 -- start start 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.760+0000 7fb3bbd49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.760+0000 7fb3bbd49700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3b407c1c0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.761+0000 7fb3b9ae5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:09.497 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.761+0000 7fb3b9ae5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40946/0 (socket says 172.21.15.40:40946) 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.761+0000 7fb3b9ae5700 1 -- 172.21.15.40:0/3640871863 learned_addr learned my addr 172.21.15.40:0/3640871863 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.761+0000 7fb3b9ae5700 1 -- 172.21.15.40:0/3640871863 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3b407a840 con 0x7fb3b407bdb0 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.761+0000 7fb3b9ae5700 1 --2- 172.21.15.40:0/3640871863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb3b00098a0 tx=0x7fb3b0009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4a3ebdf611591ddd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3b8ae3700 1 -- 172.21.15.40:0/3640871863 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb3b0004160 con 0x7fb3b407bdb0 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3b8ae3700 1 -- 172.21.15.40:0/3640871863 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3b00042c0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.498 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3b8ae3700 1 -- 172.21.15.40:0/3640871863 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb3b00119b0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.499 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3640871863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 msgr2=0x7fb3b407a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:09.499 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3640871863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7fb3b00098a0 tx=0x7fb3b0009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:09.499 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3640871863 shutdown_connections 2023-07-14T11:03:09.499 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3640871863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b407a300 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.762+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3640871863 >> 172.21.15.40:0/3640871863 conn(0x7fb3b4100910 msgr2=0x7fb3b4102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.763+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3640871863 shutdown_connections 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.763+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3640871863 wait complete. 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3bbd49700 1 Processor -- start 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3bbd49700 1 -- start start 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3bbd49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:09.500 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3bbd49700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3b419b6c0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.501 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3b9ae5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:09.503 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3b9ae5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40960/0 (socket says 172.21.15.40:40960) 2023-07-14T11:03:09.503 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.764+0000 7fb3b9ae5700 1 -- 172.21.15.40:0/3391550876 learned_addr learned my addr 172.21.15.40:0/3391550876 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:09.503 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.765+0000 7fb3b9ae5700 1 -- 172.21.15.40:0/3391550876 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb3b0009550 con 0x7fb3b407bdb0 2023-07-14T11:03:09.503 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.765+0000 7fb3b9ae5700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fb3b0000c00 tx=0x7fb3b0011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:09.503 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.765+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb3b00039f0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.765+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb3b419b8c0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.766+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3b419bce0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.766+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb3b0003b50 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.766+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fb3b001a780 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.767+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 45496+0+0 (secure 0 0 0) 0x7fb3b001a9f0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.504 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.767+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb398005320 con 0x7fb3b407bdb0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.767+0000 7fb3aaffd700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 0x7fb3a003ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.768+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fb3b004c8e0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.768+0000 7fb3b92e4700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 0x7fb3a003ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.768+0000 7fb3b92e4700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 0x7fb3a003ac60 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fb3a4006fd0 tx=0x7fb3a4006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.773+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb3b00237b0 con 0x7fb3b407bdb0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:08.995+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) v1 -- 0x7fb398000bf0 con 0x7fb3a00387b0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.075+0000 7fb3aaffd700 1 -- 172.21.15.40:0/3391550876 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+32 (secure 0 0 0) 0x7fb398000bf0 con 0x7fb3a00387b0 2023-07-14T11:03:09.505 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 msgr2=0x7fb3a003ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 0x7fb3a003ac60 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fb3a4006fd0 tx=0x7fb3a4006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 msgr2=0x7fb3b419b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fb3b0000c00 tx=0x7fb3b0011a30 comp rx=0 tx=0).stop 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 shutdown_connections 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb3a00387b0 0x7fb3a003ac60 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:09.506 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 --2- 172.21.15.40:0/3391550876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3b407bdb0 0x7fb3b419b180 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:09.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.077+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 >> 172.21.15.40:0/3391550876 conn(0x7fb3b4100910 msgr2=0x7fb3b4101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:09.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.078+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 shutdown_connections 2023-07-14T11:03:09.507 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:09.078+0000 7fb3bbd49700 1 -- 172.21.15.40:0/3391550876 wait complete. 2023-07-14T11:03:09.507 INFO:teuthology.orchestra.run.smithi040.stdout:Creating initial admin user... 2023-07-14T11:03:09.855 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:09 smithi040 ceph-mon[97954]: mgrmap e12: x(active, since 2s) 2023-07-14T11:03:09.855 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:09.855 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:09.855 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:11.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:10 smithi040 ceph-mon[97954]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:11.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:11.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:11.080 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$LpcQhCR6prTL2g4TAFTUtetJQRIaE26aRDCmXictek4Yt0vL9RZAK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1689332590, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-07-14T11:03:11.080 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.078+0000 7f0611cce700 1 Processor -- start 2023-07-14T11:03:11.080 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.078+0000 7f0611cce700 1 -- start start 2023-07-14T11:03:11.080 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.078+0000 7f0611cce700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.078+0000 7f0611cce700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f060c108190 con 0x7f060c107840 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.078+0000 7f060b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40976/0 (socket says 172.21.15.40:40976) 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060b7fe700 1 -- 172.21.15.40:0/3024156568 learned_addr learned my addr 172.21.15.40:0/3024156568 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060b7fe700 1 -- 172.21.15.40:0/3024156568 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f060c1082d0 con 0x7f060c107840 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060b7fe700 1 --2- 172.21.15.40:0/3024156568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f05fc0098a0 tx=0x7f05fc009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=65fe1ec20533daf1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:11.081 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060a7fc700 1 -- 172.21.15.40:0/3024156568 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f05fc004160 con 0x7f060c107840 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060a7fc700 1 -- 172.21.15.40:0/3024156568 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05fc0042c0 con 0x7f060c107840 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.079+0000 7f060a7fc700 1 -- 172.21.15.40:0/3024156568 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f05fc0119b0 con 0x7f060c107840 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 -- 172.21.15.40:0/3024156568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 msgr2=0x7f060c107c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 --2- 172.21.15.40:0/3024156568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f05fc0098a0 tx=0x7f05fc009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 -- 172.21.15.40:0/3024156568 shutdown_connections 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 --2- 172.21.15.40:0/3024156568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c107c50 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:11.082 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 -- 172.21.15.40:0/3024156568 >> 172.21.15.40:0/3024156568 conn(0x7f060c07a8d0 msgr2=0x7f060c07acd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.080+0000 7f0611cce700 1 -- 172.21.15.40:0/3024156568 shutdown_connections 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.081+0000 7f0611cce700 1 -- 172.21.15.40:0/3024156568 wait complete. 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.082+0000 7f0611cce700 1 Processor -- start 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.082+0000 7f0611cce700 1 -- start start 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.083+0000 7f0611cce700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:11.083 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.083+0000 7f0611cce700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f060c19b6a0 con 0x7f060c107840 2023-07-14T11:03:11.084 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.083+0000 7f060b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:11.084 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.083+0000 7f060b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40992/0 (socket says 172.21.15.40:40992) 2023-07-14T11:03:11.084 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.083+0000 7f060b7fe700 1 -- 172.21.15.40:0/1904868117 learned_addr learned my addr 172.21.15.40:0/1904868117 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:11.084 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f060b7fe700 1 -- 172.21.15.40:0/1904868117 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f05fc009550 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f060b7fe700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f05fc011980 tx=0x7f05fc011a60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f05fc003a60 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f05fc003bc0 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f060c19b8a0 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.084+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f05fc01a670 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.085+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f060c19bcc0 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.086+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 45496+0+0 (secure 0 0 0) 0x7f05fc01a8e0 con 0x7f060c107840 2023-07-14T11:03:11.085 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.086+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f060c061b40 con 0x7f060c107840 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.086+0000 7f0608ff9700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 0x7f05f403ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.086+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f05fc04cb40 con 0x7f060c107840 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.087+0000 7f060affd700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 0x7f05f403ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.087+0000 7f060affd700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 0x7f05f403ac60 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f0600006fd0 tx=0x7f0600006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.092+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f05fc0233a0 con 0x7f060c107840 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.322+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f060c061960 con 0x7f05f40387b0 2023-07-14T11:03:11.086 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.572+0000 7f0608ff9700 1 -- 172.21.15.40:0/1904868117 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+252 (secure 0 0 0) 0x7f060c061960 con 0x7f05f40387b0 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 msgr2=0x7f05f403ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 0x7f05f403ac60 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f0600006fd0 tx=0x7f0600006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 msgr2=0x7f060c19b160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f05fc011980 tx=0x7f05fc011a60 comp rx=0 tx=0).stop 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 shutdown_connections 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f05f40387b0 0x7f05f403ac60 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 --2- 172.21.15.40:0/1904868117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f060c107840 0x7f060c19b160 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:11.087 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 >> 172.21.15.40:0/1904868117 conn(0x7f060c07a8d0 msgr2=0x7f060c105750 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:11.088 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 shutdown_connections 2023-07-14T11:03:11.088 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:10.575+0000 7f0611cce700 1 -- 172.21.15.40:0/1904868117 wait complete. 2023-07-14T11:03:11.088 INFO:teuthology.orchestra.run.smithi040.stdout:Fetching dashboard port number... 2023-07-14T11:03:11.880 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:11 smithi040 ceph-mon[97954]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:12.562 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stdout 8443 2023-07-14T11:03:12.566 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.640+0000 7f75a14de700 1 Processor -- start 2023-07-14T11:03:12.566 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.640+0000 7f75a14de700 1 -- start start 2023-07-14T11:03:12.566 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.641+0000 7f75a14de700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:12.566 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.641+0000 7f75a14de700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f759c07c1c0 con 0x7f759c07bdb0 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.641+0000 7f759affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.641+0000 7f759affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40998/0 (socket says 172.21.15.40:40998) 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.641+0000 7f759affd700 1 -- 172.21.15.40:0/247322293 learned_addr learned my addr 172.21.15.40:0/247322293 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.642+0000 7f759affd700 1 -- 172.21.15.40:0/247322293 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f759c07a840 con 0x7f759c07bdb0 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.642+0000 7f759affd700 1 --2- 172.21.15.40:0/247322293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f758c0098a0 tx=0x7f758c009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=750513f16cae1896 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:12.567 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.642+0000 7f7599ffb700 1 -- 172.21.15.40:0/247322293 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f758c004160 con 0x7f759c07bdb0 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.642+0000 7f7599ffb700 1 -- 172.21.15.40:0/247322293 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f758c0042c0 con 0x7f759c07bdb0 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.642+0000 7f7599ffb700 1 -- 172.21.15.40:0/247322293 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f758c0119b0 con 0x7f759c07bdb0 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 -- 172.21.15.40:0/247322293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 msgr2=0x7f759c07a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 --2- 172.21.15.40:0/247322293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f758c0098a0 tx=0x7f758c009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 -- 172.21.15.40:0/247322293 shutdown_connections 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 --2- 172.21.15.40:0/247322293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c07a300 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 -- 172.21.15.40:0/247322293 >> 172.21.15.40:0/247322293 conn(0x7f759c100910 msgr2=0x7f759c102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:12.568 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 -- 172.21.15.40:0/247322293 shutdown_connections 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.643+0000 7f75a14de700 1 -- 172.21.15.40:0/247322293 wait complete. 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.644+0000 7f75a14de700 1 Processor -- start 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f75a14de700 1 -- start start 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f75a14de700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f75a14de700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f759c19b6c0 con 0x7f759c07bdb0 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f759affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:12.569 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f759affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:41010/0 (socket says 172.21.15.40:41010) 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.645+0000 7f759affd700 1 -- 172.21.15.40:0/2860933962 learned_addr learned my addr 172.21.15.40:0/2860933962 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f759affd700 1 -- 172.21.15.40:0/2860933962 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f758c009550 con 0x7f759c07bdb0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f759affd700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f758c000c00 tx=0x7f758c011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f758c0039f0 con 0x7f759c07bdb0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f758c003b50 con 0x7f759c07bdb0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f759c19b8c0 con 0x7f759c07bdb0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.646+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f758c01a780 con 0x7f759c07bdb0 2023-07-14T11:03:12.570 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.647+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f759c19bce0 con 0x7f759c07bdb0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.647+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f759c10b5d0 con 0x7f759c07bdb0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.648+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 45496+0+0 (secure 0 0 0) 0x7f758c01aa40 con 0x7f759c07bdb0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.648+0000 7f7583fff700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 0x7f758403ac10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.648+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f758c04ca40 con 0x7f759c07bdb0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.649+0000 7f759a7fc700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 0x7f758403ac10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.649+0000 7f759a7fc700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 0x7f758403ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7590006fd0 tx=0x7f7590006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.657+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f758c04ee60 con 0x7f759c07bdb0 2023-07-14T11:03:12.571 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.926+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) v1 -- 0x7f759c19eb40 con 0x7f759c07bdb0 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.927+0000 7f7583fff700 1 -- 172.21.15.40:0/2860933962 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) v1 ==== 112+0+5 (secure 0 0 0) 0x7f758c0237b0 con 0x7f759c07bdb0 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 msgr2=0x7f758403ac10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 0x7f758403ac10 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f7590006fd0 tx=0x7f7590006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 msgr2=0x7f759c19b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f758c000c00 tx=0x7f758c011a30 comp rx=0 tx=0).stop 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 shutdown_connections 2023-07-14T11:03:12.572 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7584038760 0x7f758403ac10 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:12.573 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 --2- 172.21.15.40:0/2860933962 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f759c07bdb0 0x7f759c19b180 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:12.573 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 >> 172.21.15.40:0/2860933962 conn(0x7f759c100910 msgr2=0x7f759c101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:12.573 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 shutdown_connections 2023-07-14T11:03:12.573 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:11.929+0000 7f75a14de700 1 -- 172.21.15.40:0/2860933962 wait complete. 2023-07-14T11:03:12.584 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-07-14T11:03:12.585 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout disabled 2023-07-14T11:03:12.602 INFO:teuthology.orchestra.run.smithi040.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-07-14T11:03:12.602 INFO:teuthology.orchestra.run.smithi040.stdout:systemctl: stdout inactive 2023-07-14T11:03:12.602 INFO:teuthology.orchestra.run.smithi040.stdout:firewalld.service is not enabled 2023-07-14T11:03:12.602 INFO:teuthology.orchestra.run.smithi040.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-07-14T11:03:12.605 INFO:teuthology.orchestra.run.smithi040.stdout:Ceph Dashboard is now available at: 2023-07-14T11:03:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: URL: https://smithi040.front.sepia.ceph.com:8443/ 2023-07-14T11:03:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: User: admin 2023-07-14T11:03:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: Password: ae80bt78px 2023-07-14T11:03:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:12.606 INFO:teuthology.orchestra.run.smithi040.stdout:Saving cluster configuration to /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config directory 2023-07-14T11:03:12.606 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling autotune for osd_memory_target 2023-07-14T11:03:12.706 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:12 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2860933962' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-07-14T11:03:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:13 smithi040 ceph-mon[97954]: mgrmap e13: x(active, since 6s) 2023-07-14T11:03:13.884 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.197+0000 7fa43a23f700 1 Processor -- start 2023-07-14T11:03:13.885 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.197+0000 7fa43a23f700 1 -- start start 2023-07-14T11:03:13.888 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.197+0000 7fa43a23f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:13.888 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.197+0000 7fa43a23f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa43407c1c0 con 0x7fa43407bdb0 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4337fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4337fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:41014/0 (socket says 172.21.15.40:41014) 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4337fe700 1 -- 172.21.15.40:0/662116187 learned_addr learned my addr 172.21.15.40:0/662116187 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4337fe700 1 -- 172.21.15.40:0/662116187 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa43407a840 con 0x7fa43407bdb0 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4337fe700 1 --2- 172.21.15.40:0/662116187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa4240098a0 tx=0x7fa424009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=49176b761cc3c3ae server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.198+0000 7fa4327fc700 1 -- 172.21.15.40:0/662116187 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa424004160 con 0x7fa43407bdb0 2023-07-14T11:03:13.889 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa4327fc700 1 -- 172.21.15.40:0/662116187 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa4240042c0 con 0x7fa43407bdb0 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa4327fc700 1 -- 172.21.15.40:0/662116187 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa4240119b0 con 0x7fa43407bdb0 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 -- 172.21.15.40:0/662116187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 msgr2=0x7fa43407a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 --2- 172.21.15.40:0/662116187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa4240098a0 tx=0x7fa424009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 -- 172.21.15.40:0/662116187 shutdown_connections 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 --2- 172.21.15.40:0/662116187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43407a300 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 -- 172.21.15.40:0/662116187 >> 172.21.15.40:0/662116187 conn(0x7fa434100910 msgr2=0x7fa434102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 -- 172.21.15.40:0/662116187 shutdown_connections 2023-07-14T11:03:13.890 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.199+0000 7fa43a23f700 1 -- 172.21.15.40:0/662116187 wait complete. 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.201+0000 7fa43a23f700 1 Processor -- start 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.201+0000 7fa43a23f700 1 -- start start 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.201+0000 7fa43a23f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.201+0000 7fa43a23f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa43419b6c0 con 0x7fa43407bdb0 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.202+0000 7fa4337fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.202+0000 7fa4337fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:41024/0 (socket says 172.21.15.40:41024) 2023-07-14T11:03:13.891 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.202+0000 7fa4337fe700 1 -- 172.21.15.40:0/2213315318 learned_addr learned my addr 172.21.15.40:0/2213315318 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.202+0000 7fa4337fe700 1 -- 172.21.15.40:0/2213315318 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa424009550 con 0x7fa43407bdb0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.202+0000 7fa4337fe700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa424000c00 tx=0x7fa424011a30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.203+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa4240039f0 con 0x7fa43407bdb0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.203+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa424003b50 con 0x7fa43407bdb0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.203+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa42401a780 con 0x7fa43407bdb0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.203+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa43419b8c0 con 0x7fa43407bdb0 2023-07-14T11:03:13.892 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.203+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa43419bce0 con 0x7fa43407bdb0 2023-07-14T11:03:13.893 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.204+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fa424024440 con 0x7fa43407bdb0 2023-07-14T11:03:13.893 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.204+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa434061b40 con 0x7fa43407bdb0 2023-07-14T11:03:13.893 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.204+0000 7fa430ff9700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 0x7fa41c03ad00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:13.893 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.204+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fa42404c980 con 0x7fa43407bdb0 2023-07-14T11:03:13.893 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.205+0000 7fa432ffd700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 0x7fa41c03ad00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:13.894 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.205+0000 7fa432ffd700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 0x7fa41c03ad00 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fa428006fd0 tx=0x7fa428006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:13.894 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.209+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa424018d90 con 0x7fa43407bdb0 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.414+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) v1 -- 0x7fa43419eb40 con 0x7fa43407bdb0 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.415+0000 7fa430ff9700 1 -- 172.21.15.40:0/2213315318 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) v1 ==== 127+0+0 (secure 0 0 0) 0x7fa424018350 con 0x7fa43407bdb0 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 msgr2=0x7fa41c03ad00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 0x7fa41c03ad00 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fa428006fd0 tx=0x7fa428006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 msgr2=0x7fa43419b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:13.895 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fa424000c00 tx=0x7fa424011a30 comp rx=0 tx=0).stop 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 shutdown_connections 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa41c038850 0x7fa41c03ad00 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 --2- 172.21.15.40:0/2213315318 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa43407bdb0 0x7fa43419b180 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 >> 172.21.15.40:0/2213315318 conn(0x7fa434100910 msgr2=0x7fa434101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.417+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 shutdown_connections 2023-07-14T11:03:13.896 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:13.418+0000 7fa43a23f700 1 -- 172.21.15.40:0/2213315318 wait complete. 2023-07-14T11:03:15.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.430+0000 7f568cbeb700 1 Processor -- start 2023-07-14T11:03:15.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568cbeb700 1 -- start start 2023-07-14T11:03:15.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568cbeb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:15.272 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568cbeb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f568807c1c0 con 0x7f568807bdb0 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51216/0 (socket says 172.21.15.40:51216) 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.431+0000 7f568659c700 1 -- 172.21.15.40:0/2710615450 learned_addr learned my addr 172.21.15.40:0/2710615450 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.432+0000 7f568659c700 1 -- 172.21.15.40:0/2710615450 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f568807a840 con 0x7f568807bdb0 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.432+0000 7f568659c700 1 --2- 172.21.15.40:0/2710615450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f56780098a0 tx=0x7f5678009bb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a0656ac5c2f69f4e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.432+0000 7f568559a700 1 -- 172.21.15.40:0/2710615450 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5678004160 con 0x7f568807bdb0 2023-07-14T11:03:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.432+0000 7f568559a700 1 -- 172.21.15.40:0/2710615450 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f56780042c0 con 0x7f568807bdb0 2023-07-14T11:03:15.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.432+0000 7f568559a700 1 -- 172.21.15.40:0/2710615450 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56780119b0 con 0x7f568807bdb0 2023-07-14T11:03:15.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 -- 172.21.15.40:0/2710615450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 msgr2=0x7f568807a300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:15.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 --2- 172.21.15.40:0/2710615450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f56780098a0 tx=0x7f5678009bb0 comp rx=0 tx=0).stop 2023-07-14T11:03:15.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 -- 172.21.15.40:0/2710615450 shutdown_connections 2023-07-14T11:03:15.274 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 --2- 172.21.15.40:0/2710615450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568807a300 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:15.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 -- 172.21.15.40:0/2710615450 >> 172.21.15.40:0/2710615450 conn(0x7f5688100910 msgr2=0x7f5688102d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:15.275 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 -- 172.21.15.40:0/2710615450 shutdown_connections 2023-07-14T11:03:15.276 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.433+0000 7f568cbeb700 1 -- 172.21.15.40:0/2710615450 wait complete. 2023-07-14T11:03:15.276 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.434+0000 7f568cbeb700 1 Processor -- start 2023-07-14T11:03:15.277 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.434+0000 7f568cbeb700 1 -- start start 2023-07-14T11:03:15.277 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.434+0000 7f568cbeb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:15.277 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.434+0000 7f568cbeb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f568819b6c0 con 0x7f568807bdb0 2023-07-14T11:03:15.277 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.435+0000 7f568659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.435+0000 7f568659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51232/0 (socket says 172.21.15.40:51232) 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.435+0000 7f568659c700 1 -- 172.21.15.40:0/1858884339 learned_addr learned my addr 172.21.15.40:0/1858884339 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.435+0000 7f568659c700 1 -- 172.21.15.40:0/1858884339 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5678009550 con 0x7f568807bdb0 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.435+0000 7f568659c700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f5678003e60 tx=0x7f5678011aa0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.436+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f5678003970 con 0x7f568807bdb0 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.436+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5678003ad0 con 0x7f568807bdb0 2023-07-14T11:03:15.278 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.436+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f568819b8c0 con 0x7f568807bdb0 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.436+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f56780195e0 con 0x7f568807bdb0 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.436+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f568819bce0 con 0x7f568807bdb0 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.437+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5688061b40 con 0x7f568807bdb0 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.437+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f5678019850 con 0x7f568807bdb0 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.438+0000 7f56777fe700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 0x7f567003acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:15.279 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.438+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f567804cac0 con 0x7f568807bdb0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.438+0000 7f5685d9b700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 0x7f567003acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.439+0000 7f5685d9b700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 0x7f567003acb0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f567c006fd0 tx=0x7f567c006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.443+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5678003c40 con 0x7f568807bdb0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.761+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) v1 -- 0x7f568819eb40 con 0x7f568807bdb0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.764+0000 7f56777fe700 1 -- 172.21.15.40:0/1858884339 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v28)=0 set mgr/dashboard/cluster/status v28) v1 ==== 153+0+0 (secure 0 0 0) 0x7f5678018350 con 0x7f568807bdb0 2023-07-14T11:03:15.280 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 msgr2=0x7f567003acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:15.281 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 0x7f567003acb0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f567c006fd0 tx=0x7f567c006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:15.281 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 msgr2=0x7f568819b180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:15.281 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f5678003e60 tx=0x7f5678011aa0 comp rx=0 tx=0).stop 2023-07-14T11:03:15.281 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 shutdown_connections 2023-07-14T11:03:15.281 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5670038800 0x7f567003acb0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:15.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 --2- 172.21.15.40:0/1858884339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f568807bdb0 0x7f568819b180 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:15.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 >> 172.21.15.40:0/1858884339 conn(0x7f5688100910 msgr2=0x7f5688101f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:15.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 shutdown_connections 2023-07-14T11:03:15.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr 2023-07-14T11:03:14.766+0000 7f568cbeb700 1 -- 172.21.15.40:0/1858884339 wait complete. 2023-07-14T11:03:15.282 INFO:teuthology.orchestra.run.smithi040.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-07-14T11:03:15.283 INFO:teuthology.orchestra.run.smithi040.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-07-14T11:03:15.283 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.283 INFO:teuthology.orchestra.run.smithi040.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:15.283 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.283 INFO:teuthology.orchestra.run.smithi040.stdout:Or, if you are only running a single cluster on this host: 2023-07-14T11:03:15.284 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.284 INFO:teuthology.orchestra.run.smithi040.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-07-14T11:03:15.284 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.284 INFO:teuthology.orchestra.run.smithi040.stdout:Please consider enabling telemetry to help improve Ceph: 2023-07-14T11:03:15.285 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.285 INFO:teuthology.orchestra.run.smithi040.stdout: ceph telemetry on 2023-07-14T11:03:15.285 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.285 INFO:teuthology.orchestra.run.smithi040.stdout:For more information see: 2023-07-14T11:03:15.286 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.287 INFO:teuthology.orchestra.run.smithi040.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2023-07-14T11:03:15.287 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:15.288 INFO:teuthology.orchestra.run.smithi040.stdout:Bootstrap complete. 2023-07-14T11:03:15.332 INFO:tasks.cephadm:Fetching config... 2023-07-14T11:03:15.332 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:03:15.333 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-07-14T11:03:15.377 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-07-14T11:03:15.377 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:03:15.378 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-07-14T11:03:15.469 INFO:tasks.cephadm:Fetching mon keyring... 2023-07-14T11:03:15.469 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:03:15.470 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/keyring of=/dev/stdout 2023-07-14T11:03:15.581 INFO:tasks.cephadm:Fetching pub ssh key... 2023-07-14T11:03:15.582 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:03:15.582 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-07-14T11:03:15.668 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-07-14T11:03:15.669 DEBUG:teuthology.orchestra.run.smithi040:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-07-14T11:03:15.816 INFO:teuthology.orchestra.run.smithi040.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:15.835 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:15 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1858884339' entity='client.admin' 2023-07-14T11:03:15.845 DEBUG:teuthology.orchestra.run.smithi045:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-07-14T11:03:15.891 INFO:teuthology.orchestra.run.smithi045.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:15.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-07-14T11:03:15.952 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC5yU2Q3IOCt3k6XcPIy7HCOBkW9SGdGn60Zomv3YsfhGm5AbhF8X88hw5bzQkl1ddA5aCsbPRqWJsJMiPnkyqwh2ItAxeEAsNCqajAchZ2n2JGVjXATsJEL05X4ePc4KNPjtNx55bpbiz7eB0BRSrMxlE4TAGy8AZlCtklwz3wZLkP47xYHHjrLg2dLIvtwOgnM2kcxihveMipF9VRWS4bNhO5QsrNz7liOqqxHR3WTDMeUuB0T5gg5+IXxksAmtPOF4Hw1FVNoD2KntHwdLAGmiUwqj8/0XdT6DlwGsxBEI7+pcNOdwsrmlZySQ0wK6cMQSUO5+tmqbioGjHVab94K5O3uCHgvJS7sRn2wi4u5ss/6dqxvzsIN/dbvlbJKHFmRD/xcymlS7/nvy9Fysb0pgvcH6k7roi7M63c0Xm6ixjFoVX6NrAxJN3/sDOsCYUGEpDcd7/8SakhdnkGuRckkGH3jd1eRGdm+LSrNV4HAd/w0U4co/zII4vINHWQ1FM= ceph-bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:15.965 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-07-14T11:03:17.070 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi040", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:18 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:18.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.731+0000 7f49a86e5700 1 -- 172.21.15.40:0/3682405897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 msgr2=0x7f49a00ff9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:18.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.731+0000 7f49a86e5700 1 --2- 172.21.15.40:0/3682405897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a00ff9f0 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f499c009910 tx=0x7f499c009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:18.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.732+0000 7f49a86e5700 1 -- 172.21.15.40:0/3682405897 shutdown_connections 2023-07-14T11:03:18.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.732+0000 7f49a86e5700 1 --2- 172.21.15.40:0/3682405897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a00ff9f0 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:18.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.732+0000 7f49a86e5700 1 -- 172.21.15.40:0/3682405897 >> 172.21.15.40:0/3682405897 conn(0x7f49a00fab90 msgr2=0x7f49a00fcfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:18.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.732+0000 7f49a86e5700 1 -- 172.21.15.40:0/3682405897 shutdown_connections 2023-07-14T11:03:18.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.732+0000 7f49a86e5700 1 -- 172.21.15.40:0/3682405897 wait complete. 2023-07-14T11:03:18.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.733+0000 7f49a86e5700 1 Processor -- start 2023-07-14T11:03:18.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.733+0000 7f49a86e5700 1 -- start start 2023-07-14T11:03:18.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.733+0000 7f49a86e5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:18.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.733+0000 7f49a86e5700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49a0196e80 con 0x7f49a00ff5e0 2023-07-14T11:03:18.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a6481700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:18.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a6481700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51262/0 (socket says 172.21.15.40:51262) 2023-07-14T11:03:18.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a6481700 1 -- 172.21.15.40:0/4270828243 learned_addr learned my addr 172.21.15.40:0/4270828243 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:18.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a6481700 1 -- 172.21.15.40:0/4270828243 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f499c0095f0 con 0x7f49a00ff5e0 2023-07-14T11:03:18.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a6481700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f499c004f40 tx=0x7f499c0057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:18.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f499c01d070 con 0x7f49a00ff5e0 2023-07-14T11:03:18.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f499c00bc10 con 0x7f49a00ff5e0 2023-07-14T11:03:18.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f49a0197080 con 0x7f49a00ff5e0 2023-07-14T11:03:18.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.734+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f499c003a70 con 0x7f49a00ff5e0 2023-07-14T11:03:18.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.735+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49a0197520 con 0x7f49a00ff5e0 2023-07-14T11:03:18.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.736+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f499c003ce0 con 0x7f49a00ff5e0 2023-07-14T11:03:18.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.736+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f49a01905b0 con 0x7f49a00ff5e0 2023-07-14T11:03:18.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.736+0000 7f49977fe700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 0x7f498c03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:18.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.736+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f499c04c530 con 0x7f49a00ff5e0 2023-07-14T11:03:18.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.736+0000 7f49a5c80700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 0x7f498c03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:18.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.737+0000 7f49a5c80700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 0x7f498c03acb0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4990006fd0 tx=0x7f4990006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:18.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.741+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f499c02abb0 con 0x7f49a00ff5e0 2023-07-14T11:03:18.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.946+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) v1 -- 0x7f49a002c850 con 0x7f49a00ff5e0 2023-07-14T11:03:18.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.949+0000 7f49977fe700 1 -- 172.21.15.40:0/4270828243 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) v1 ==== 125+0+0 (secure 0 0 0) 0x7f499c026070 con 0x7f49a00ff5e0 2023-07-14T11:03:18.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 msgr2=0x7f498c03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:18.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 0x7f498c03acb0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f4990006fd0 tx=0x7f4990006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:18.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 msgr2=0x7f49a0196940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:18.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f499c004f40 tx=0x7f499c0057c0 comp rx=0 tx=0).stop 2023-07-14T11:03:18.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 shutdown_connections 2023-07-14T11:03:18.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f498c038800 0x7f498c03acb0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:18.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 --2- 172.21.15.40:0/4270828243 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49a00ff5e0 0x7f49a0196940 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:18.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 >> 172.21.15.40:0/4270828243 conn(0x7f49a00fab90 msgr2=0x7f49a00fcfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:18.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.952+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 shutdown_connections 2023-07-14T11:03:18.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:18.953+0000 7f49a86e5700 1 -- 172.21.15.40:0/4270828243 wait complete. 2023-07-14T11:03:19.451 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-07-14T11:03:19.452 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-07-14T11:03:20.181 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:19 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4270828243' entity='client.admin' 2023-07-14T11:03:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:21.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/97805543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 msgr2=0x7fbdf40ff9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:21.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/97805543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf40ff9f0 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fbdf0009910 tx=0x7fbdf0009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:21.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/97805543 shutdown_connections 2023-07-14T11:03:21.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/97805543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf40ff9f0 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:21.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/97805543 >> 172.21.15.40:0/97805543 conn(0x7fbdf40fab90 msgr2=0x7fbdf40fcfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:21.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/97805543 shutdown_connections 2023-07-14T11:03:21.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.659+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/97805543 wait complete. 2023-07-14T11:03:21.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.660+0000 7fbdfb3b6700 1 Processor -- start 2023-07-14T11:03:21.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.660+0000 7fbdfb3b6700 1 -- start start 2023-07-14T11:03:21.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdfb3b6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:21.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdfb3b6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbdf4194c90 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdf9152700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:21.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdf9152700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51280/0 (socket says 172.21.15.40:51280) 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdf9152700 1 -- 172.21.15.40:0/2213040481 learned_addr learned my addr 172.21.15.40:0/2213040481 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdf9152700 1 -- 172.21.15.40:0/2213040481 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbdf00095f0 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.661+0000 7fbdf9152700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fbdf0004f40 tx=0x7fbdf00057c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.662+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbdf001d070 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.662+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbdf000bc10 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.662+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbdf4194e90 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.662+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fbdf0003a70 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.662+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbdf4195330 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.663+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fbdf0003d30 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.663+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbdf418e1b0 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.663+0000 7fbdea7fc700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 0x7fbde003ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.663+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fbdf004c570 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.664+0000 7fbdf8951700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 0x7fbde003ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:21.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.664+0000 7fbdf8951700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 0x7fbde003ac60 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fbde4006fd0 tx=0x7fbde4006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:21.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.669+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fbdf002ac00 con 0x7fbdf40ff5e0 2023-07-14T11:03:21.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.877+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) v1 -- 0x7fbdf40609e0 con 0x7fbde00387b0 2023-07-14T11:03:21.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.881+0000 7fbdea7fc700 1 -- 172.21.15.40:0/2213040481 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7fbdf40609e0 con 0x7fbde00387b0 2023-07-14T11:03:21.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 msgr2=0x7fbde003ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:21.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 0x7fbde003ac60 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fbde4006fd0 tx=0x7fbde4006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:21.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 msgr2=0x7fbdf4194750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:21.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fbdf0004f40 tx=0x7fbdf00057c0 comp rx=0 tx=0).stop 2023-07-14T11:03:21.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 shutdown_connections 2023-07-14T11:03:21.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbde00387b0 0x7fbde003ac60 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:21.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 --2- 172.21.15.40:0/2213040481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbdf40ff5e0 0x7fbdf4194750 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:21.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 >> 172.21.15.40:0/2213040481 conn(0x7fbdf40fab90 msgr2=0x7fbdf40fb8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:21.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 shutdown_connections 2023-07-14T11:03:21.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:21.884+0000 7fbdfb3b6700 1 -- 172.21.15.40:0/2213040481 wait complete. 2023-07-14T11:03:22.359 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi045 2023-07-14T11:03:22.359 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:03:22.360 DEBUG:teuthology.orchestra.run.smithi045:> dd of=/etc/ceph/ceph.conf 2023-07-14T11:03:22.381 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:03:22.382 DEBUG:teuthology.orchestra.run.smithi045:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:22.441 INFO:tasks.cephadm:Adding host smithi045 to orchestrator... 2023-07-14T11:03:22.441 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch host add smithi045 2023-07-14T11:03:23.192 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:23.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:23.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:23.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:23.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:23.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:23.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:22 smithi040 ceph-mon[97954]: Updating smithi040:/etc/ceph/ceph.conf 2023-07-14T11:03:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:23 smithi040 ceph-mon[97954]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:24.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 -- 172.21.15.40:0/1279453279 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 msgr2=0x7fca3c1048c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:24.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 --2- 172.21.15.40:0/1279453279 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1048c0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fca38009910 tx=0x7fca38009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:24.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 -- 172.21.15.40:0/1279453279 shutdown_connections 2023-07-14T11:03:24.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 --2- 172.21.15.40:0/1279453279 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1048c0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:24.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 -- 172.21.15.40:0/1279453279 >> 172.21.15.40:0/1279453279 conn(0x7fca3c0742b0 msgr2=0x7fca3c0746b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:24.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 -- 172.21.15.40:0/1279453279 shutdown_connections 2023-07-14T11:03:24.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.674+0000 7fca43ed5700 1 -- 172.21.15.40:0/1279453279 wait complete. 2023-07-14T11:03:24.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.675+0000 7fca43ed5700 1 Processor -- start 2023-07-14T11:03:24.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca43ed5700 1 -- start start 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca43ed5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca43ed5700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca3c102040 con 0x7fca3c1044f0 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca41c71700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca41c71700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:57356/0 (socket says 172.21.15.40:57356) 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca41c71700 1 -- 172.21.15.40:0/3963118322 learned_addr learned my addr 172.21.15.40:0/3963118322 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.676+0000 7fca41c71700 1 -- 172.21.15.40:0/3963118322 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca380095f0 con 0x7fca3c1044f0 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca41c71700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fca38006010 tx=0x7fca38004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:24.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca3801d070 con 0x7fca3c1044f0 2023-07-14T11:03:24.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fca3800bc10 con 0x7fca3c1044f0 2023-07-14T11:03:24.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca3c100600 con 0x7fca3c1044f0 2023-07-14T11:03:24.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fca38003910 con 0x7fca3c1044f0 2023-07-14T11:03:24.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.677+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca3c100a80 con 0x7fca3c1044f0 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.678+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fca3800bd80 con 0x7fca3c1044f0 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.678+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca3c061b40 con 0x7fca3c1044f0 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.679+0000 7fca32ffd700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 0x7fca2803acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.679+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fca3804c5a0 con 0x7fca3c1044f0 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.679+0000 7fca41470700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 0x7fca2803acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:24.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.679+0000 7fca41470700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 0x7fca2803acb0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fca2c006fd0 tx=0x7fca2c006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:24.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.684+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fca38029360 con 0x7fca3c1044f0 2023-07-14T11:03:24.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:24.895+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi045", "target": ["mon-mgr", ""]}) v1 -- 0x7fca3c100dd0 con 0x7fca28038800 2023-07-14T11:03:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:24 smithi040 ceph-mon[97954]: Updating smithi040:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:24 smithi040 ceph-mon[97954]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:25 smithi040 ceph-mon[97954]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi045", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:27.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:26 smithi040 ceph-mon[97954]: Deploying cephadm binary to smithi045 2023-07-14T11:03:27.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.316+0000 7fca32ffd700 1 -- 172.21.15.40:0/3963118322 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+48 (secure 0 0 0) 0x7fca3c100dd0 con 0x7fca28038800 2023-07-14T11:03:27.319 INFO:teuthology.orchestra.run.smithi040.stdout:Added host 'smithi045' with addr '172.21.15.45' 2023-07-14T11:03:27.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.318+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 msgr2=0x7fca2803acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:27.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.318+0000 7fca43ed5700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 0x7fca2803acb0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fca2c006fd0 tx=0x7fca2c006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:27.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 msgr2=0x7fca3c1000c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:27.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fca38006010 tx=0x7fca38004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:27.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 shutdown_connections 2023-07-14T11:03:27.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fca28038800 0x7fca2803acb0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:27.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 --2- 172.21.15.40:0/3963118322 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fca3c1044f0 0x7fca3c1000c0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:27.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 >> 172.21.15.40:0/3963118322 conn(0x7fca3c0742b0 msgr2=0x7fca3c067fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:27.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 shutdown_connections 2023-07-14T11:03:27.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:27.319+0000 7fca43ed5700 1 -- 172.21.15.40:0/3963118322 wait complete. 2023-07-14T11:03:27.780 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch host ls --format=json 2023-07-14T11:03:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:27 smithi040 ceph-mon[97954]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:28.510 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:28 smithi040 ceph-mon[97954]: Added host smithi045 2023-07-14T11:03:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:30.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 -- 172.21.15.40:0/2478481775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 msgr2=0x7f21b0105a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:30.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 --2- 172.21.15.40:0/2478481775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b0105a00 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f21a0009910 tx=0x7f21a0009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 -- 172.21.15.40:0/2478481775 shutdown_connections 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 --2- 172.21.15.40:0/2478481775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b0105a00 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 -- 172.21.15.40:0/2478481775 >> 172.21.15.40:0/2478481775 conn(0x7f21b00fb130 msgr2=0x7f21b00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 -- 172.21.15.40:0/2478481775 shutdown_connections 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.004+0000 7f21b530e700 1 -- 172.21.15.40:0/2478481775 wait complete. 2023-07-14T11:03:30.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.005+0000 7f21b530e700 1 Processor -- start 2023-07-14T11:03:30.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.005+0000 7f21b530e700 1 -- start start 2023-07-14T11:03:30.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21b530e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:30.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21b530e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21b019ce70 con 0x7f21b0105630 2023-07-14T11:03:30.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21aeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:30.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21aeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:57378/0 (socket says 172.21.15.40:57378) 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21aeffd700 1 -- 172.21.15.40:0/3478410086 learned_addr learned my addr 172.21.15.40:0/3478410086 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.006+0000 7f21aeffd700 1 -- 172.21.15.40:0/3478410086 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21a00095f0 con 0x7f21b0105630 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f21aeffd700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f21a0006010 tx=0x7f21a0004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f21a001d070 con 0x7f21b0105630 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f21a000bc10 con 0x7f21b0105630 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21b019d0d0 con 0x7f21b0105630 2023-07-14T11:03:30.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f21a0003910 con 0x7f21b0105630 2023-07-14T11:03:30.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.007+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21b01969b0 con 0x7f21b0105630 2023-07-14T11:03:30.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.008+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f21a000bd80 con 0x7f21b0105630 2023-07-14T11:03:30.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.008+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f21b0061b40 con 0x7f21b0105630 2023-07-14T11:03:30.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.009+0000 7f2197fff700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 0x7f219803acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:30.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.009+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f21a004c4a0 con 0x7f21b0105630 2023-07-14T11:03:30.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.009+0000 7f21ae7fc700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 0x7f219803acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:30.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.010+0000 7f21ae7fc700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 0x7f219803acb0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f21a4006fd0 tx=0x7f21a4006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:30.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.014+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f21a002ad00 con 0x7f21b0105630 2023-07-14T11:03:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:29 smithi040 ceph-mon[97954]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:30.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.227+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f21b0197470 con 0x7f2198038800 2023-07-14T11:03:30.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.229+0000 7f2197fff700 1 -- 172.21.15.40:0/3478410086 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+159 (secure 0 0 0) 0x7f21b0197470 con 0x7f2198038800 2023-07-14T11:03:30.232 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:30.232 INFO:teuthology.orchestra.run.smithi040.stdout:[{"addr": "172.21.15.40", "hostname": "smithi040", "labels": [], "status": ""}, {"addr": "172.21.15.45", "hostname": "smithi045", "labels": [], "status": ""}] 2023-07-14T11:03:30.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 msgr2=0x7f219803acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:30.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 0x7f219803acb0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f21a4006fd0 tx=0x7f21a4006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:30.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 msgr2=0x7f21b019c930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:30.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f21a0006010 tx=0x7f21a0004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:30.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 shutdown_connections 2023-07-14T11:03:30.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2198038800 0x7f219803acb0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:30.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 --2- 172.21.15.40:0/3478410086 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21b0105630 0x7f21b019c930 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:30.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 >> 172.21.15.40:0/3478410086 conn(0x7f21b00fb130 msgr2=0x7f21b0074200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:30.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 shutdown_connections 2023-07-14T11:03:30.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:30.231+0000 7f21b530e700 1 -- 172.21.15.40:0/3478410086 wait complete. 2023-07-14T11:03:30.687 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi130 2023-07-14T11:03:30.687 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:03:30.687 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.conf 2023-07-14T11:03:30.708 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:03:30.709 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:30.768 INFO:tasks.cephadm:Adding host smithi130 to orchestrator... 2023-07-14T11:03:30.768 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch host add smithi130 2023-07-14T11:03:31.498 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:32.111 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:31 smithi040 ceph-mon[97954]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:03:32.111 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:31 smithi040 ceph-mon[97954]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 -- 172.21.15.40:0/3657785807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 msgr2=0x7fc044107bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 --2- 172.21.15.40:0/3657785807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044107bd0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc034009910 tx=0x7fc034009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 -- 172.21.15.40:0/3657785807 shutdown_connections 2023-07-14T11:03:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 --2- 172.21.15.40:0/3657785807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044107bd0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 -- 172.21.15.40:0/3657785807 >> 172.21.15.40:0/3657785807 conn(0x7fc0440fd360 msgr2=0x7fc0440ff770 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:33.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 -- 172.21.15.40:0/3657785807 shutdown_connections 2023-07-14T11:03:33.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.140+0000 7fc049c32700 1 -- 172.21.15.40:0/3657785807 wait complete. 2023-07-14T11:03:33.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.141+0000 7fc049c32700 1 Processor -- start 2023-07-14T11:03:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.141+0000 7fc049c32700 1 -- start start 2023-07-14T11:03:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc049c32700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc049c32700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc044197200 con 0x7fc044107800 2023-07-14T11:03:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc0437fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc0437fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:57388/0 (socket says 172.21.15.40:57388) 2023-07-14T11:03:33.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc0437fe700 1 -- 172.21.15.40:0/665211052 learned_addr learned my addr 172.21.15.40:0/665211052 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:33.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.142+0000 7fc0437fe700 1 -- 172.21.15.40:0/665211052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0340095f0 con 0x7fc044107800 2023-07-14T11:03:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc0437fe700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fc0340092e0 tx=0x7fc034004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc03401d070 con 0x7fc044107800 2023-07-14T11:03:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc03400bc10 con 0x7fc044107800 2023-07-14T11:03:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc044197400 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fc034003930 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.143+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0441978a0 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.144+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fc034003b60 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.145+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc044061b40 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.145+0000 7fc040ff9700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 0x7fc02c03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.145+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fc03404c6d0 con 0x7fc044107800 2023-07-14T11:03:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.145+0000 7fc042ffd700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 0x7fc02c03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:33.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.146+0000 7fc042ffd700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 0x7fc02c03acb0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fc038006fd0 tx=0x7fc038006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.151+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc03400fd00 con 0x7fc044107800 2023-07-14T11:03:33.372 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:33.369+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}) v1 -- 0x7fc0441066a0 con 0x7fc02c038800 2023-07-14T11:03:33.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:33.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:33.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:33.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:33.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:33.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:34.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:34 smithi040 ceph-mon[97954]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:34.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:34 smithi040 ceph-mon[97954]: Deploying cephadm binary to smithi130 2023-07-14T11:03:35.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:35 smithi040 ceph-mon[97954]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:35.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.723+0000 7fc040ff9700 1 -- 172.21.15.40:0/665211052 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7fc0441066a0 con 0x7fc02c038800 2023-07-14T11:03:35.728 INFO:teuthology.orchestra.run.smithi040.stdout:Added host 'smithi130' with addr '172.21.15.130' 2023-07-14T11:03:35.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 msgr2=0x7fc02c03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:35.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 0x7fc02c03acb0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fc038006fd0 tx=0x7fc038006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:35.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 msgr2=0x7fc044196cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:35.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7fc0340092e0 tx=0x7fc034004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 shutdown_connections 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc02c038800 0x7fc02c03acb0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 --2- 172.21.15.40:0/665211052 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc044107800 0x7fc044196cc0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 >> 172.21.15.40:0/665211052 conn(0x7fc0440fd360 msgr2=0x7fc0440fdfc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 shutdown_connections 2023-07-14T11:03:35.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:35.727+0000 7fc049c32700 1 -- 172.21.15.40:0/665211052 wait complete. 2023-07-14T11:03:36.183 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch host ls --format=json 2023-07-14T11:03:36.913 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:37.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:37.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:36 smithi040 ceph-mon[97954]: Added host smithi130 2023-07-14T11:03:37.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:37.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:37 smithi040 ceph-mon[97954]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 -- 172.21.15.40:0/1310371392 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 msgr2=0x7f41f4105a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 --2- 172.21.15.40:0/1310371392 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4105a00 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f41e4009910 tx=0x7f41e4009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 -- 172.21.15.40:0/1310371392 shutdown_connections 2023-07-14T11:03:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 --2- 172.21.15.40:0/1310371392 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4105a00 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 -- 172.21.15.40:0/1310371392 >> 172.21.15.40:0/1310371392 conn(0x7f41f40fb130 msgr2=0x7f41f40fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 -- 172.21.15.40:0/1310371392 shutdown_connections 2023-07-14T11:03:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.438+0000 7f41f9f42700 1 -- 172.21.15.40:0/1310371392 wait complete. 2023-07-14T11:03:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.440+0000 7f41f9f42700 1 Processor -- start 2023-07-14T11:03:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.440+0000 7f41f9f42700 1 -- start start 2023-07-14T11:03:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f9f42700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f9f42700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41f4194eb0 con 0x7f41f4105630 2023-07-14T11:03:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49142/0 (socket says 172.21.15.40:49142) 2023-07-14T11:03:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f37fe700 1 -- 172.21.15.40:0/2821111156 learned_addr learned my addr 172.21.15.40:0/2821111156 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.441+0000 7f41f37fe700 1 -- 172.21.15.40:0/2821111156 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41e40095f0 con 0x7f41f4105630 2023-07-14T11:03:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f37fe700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f41e4006010 tx=0x7f41e4004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f41e401d070 con 0x7f41f4105630 2023-07-14T11:03:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f41f4195110 con 0x7f41f4105630 2023-07-14T11:03:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f41e400bc10 con 0x7f41f4105630 2023-07-14T11:03:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f41e4003910 con 0x7f41f4105630 2023-07-14T11:03:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.442+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41f4195610 con 0x7f41f4105630 2023-07-14T11:03:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.443+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f41f4061b40 con 0x7f41f4105630 2023-07-14T11:03:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.443+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f41e400bd80 con 0x7f41f4105630 2023-07-14T11:03:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.444+0000 7f41f0ff9700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 0x7f41dc03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.444+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f41e404c3f0 con 0x7f41f4105630 2023-07-14T11:03:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.444+0000 7f41f2ffd700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 0x7f41dc03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:38.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.444+0000 7f41f2ffd700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 0x7f41dc03acb0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f41e8006fd0 tx=0x7f41e8006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:38.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.449+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f41e4029360 con 0x7f41f4105630 2023-07-14T11:03:38.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.661+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f41f4195610 con 0x7f41dc038800 2023-07-14T11:03:38.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.662+0000 7f41f0ff9700 1 -- 172.21.15.40:0/2821111156 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+239 (secure 0 0 0) 0x7f41f4195610 con 0x7f41dc038800 2023-07-14T11:03:38.666 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:03:38.666 INFO:teuthology.orchestra.run.smithi040.stdout:[{"addr": "172.21.15.40", "hostname": "smithi040", "labels": [], "status": ""}, {"addr": "172.21.15.45", "hostname": "smithi045", "labels": [], "status": ""}, {"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}] 2023-07-14T11:03:38.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 msgr2=0x7f41dc03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:38.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 0x7f41dc03acb0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f41e8006fd0 tx=0x7f41e8006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 msgr2=0x7f41f4194970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f41e4006010 tx=0x7f41e4004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 shutdown_connections 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f41dc038800 0x7f41dc03acb0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 --2- 172.21.15.40:0/2821111156 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41f4105630 0x7f41f4194970 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 >> 172.21.15.40:0/2821111156 conn(0x7f41f40fb130 msgr2=0x7f41f40fbd90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 shutdown_connections 2023-07-14T11:03:38.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:38.665+0000 7f41f9f42700 1 -- 172.21.15.40:0/2821111156 wait complete. 2023-07-14T11:03:39.090 INFO:tasks.cephadm:Setting crush tunables to default 2023-07-14T11:03:39.091 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd crush tunables default 2023-07-14T11:03:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:39.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:39 smithi040 ceph-mon[97954]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:39.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:39 smithi040 ceph-mon[97954]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:03:39.827 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:03:41.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.422+0000 7fa81a53e700 1 -- 172.21.15.40:0/4101154132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 msgr2=0x7fa814072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:41.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.422+0000 7fa81a53e700 1 --2- 172.21.15.40:0/4101154132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa814072c80 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fa804009910 tx=0x7fa804009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:41.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.422+0000 7fa81a53e700 1 -- 172.21.15.40:0/4101154132 shutdown_connections 2023-07-14T11:03:41.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.422+0000 7fa81a53e700 1 --2- 172.21.15.40:0/4101154132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa814072c80 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:41.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.422+0000 7fa81a53e700 1 -- 172.21.15.40:0/4101154132 >> 172.21.15.40:0/4101154132 conn(0x7fa8140fb130 msgr2=0x7fa8140fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:41.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.423+0000 7fa81a53e700 1 -- 172.21.15.40:0/4101154132 shutdown_connections 2023-07-14T11:03:41.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.423+0000 7fa81a53e700 1 -- 172.21.15.40:0/4101154132 wait complete. 2023-07-14T11:03:41.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa81a53e700 1 Processor -- start 2023-07-14T11:03:41.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa81a53e700 1 -- start start 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa81a53e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa81a53e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa81419b670 con 0x7fa8140728b0 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa813fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa813fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49152/0 (socket says 172.21.15.40:49152) 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.424+0000 7fa813fff700 1 -- 172.21.15.40:0/2034251137 learned_addr learned my addr 172.21.15.40:0/2034251137 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa813fff700 1 -- 172.21.15.40:0/2034251137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8040095f0 con 0x7fa8140728b0 2023-07-14T11:03:41.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa813fff700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fa804006010 tx=0x7fa804004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:41.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa80401d070 con 0x7fa8140728b0 2023-07-14T11:03:41.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa8040044a0 con 0x7fa8140728b0 2023-07-14T11:03:41.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fa804003a70 con 0x7fa8140728b0 2023-07-14T11:03:41.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.425+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa81419b870 con 0x7fa8140728b0 2023-07-14T11:03:41.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.426+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa81419bc90 con 0x7fa8140728b0 2023-07-14T11:03:41.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.426+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa814061b40 con 0x7fa8140728b0 2023-07-14T11:03:41.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.427+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fa804003c60 con 0x7fa8140728b0 2023-07-14T11:03:41.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.427+0000 7fa8117fa700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 0x7fa7fc03acb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:41.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.427+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fa80404c660 con 0x7fa8140728b0 2023-07-14T11:03:41.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.427+0000 7fa8137fe700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 0x7fa7fc03acb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:41.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.428+0000 7fa8137fe700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 0x7fa7fc03acb0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fa808006fd0 tx=0x7fa808006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:41.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.432+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa804005c00 con 0x7fa8140728b0 2023-07-14T11:03:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:41.654+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7fa814102510 con 0x7fa8140728b0 2023-07-14T11:03:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:42.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.645+0000 7fa8117fa700 1 -- 172.21.15.40:0/2034251137 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) v1 ==== 124+0+0 (secure 0 0 0) 0x7fa804026020 con 0x7fa8140728b0 2023-07-14T11:03:42.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.648+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 msgr2=0x7fa7fc03acb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:42.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.648+0000 7fa81a53e700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 0x7fa7fc03acb0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fa808006fd0 tx=0x7fa808006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:42.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.648+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 msgr2=0x7fa81419b130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:42.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.648+0000 7fa81a53e700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fa804006010 tx=0x7fa804004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:42.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 shutdown_connections 2023-07-14T11:03:42.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7fc038800 0x7fa7fc03acb0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:42.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 --2- 172.21.15.40:0/2034251137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8140728b0 0x7fa81419b130 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:42.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 >> 172.21.15.40:0/2034251137 conn(0x7fa8140fb130 msgr2=0x7fa8140fbd90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:42.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 shutdown_connections 2023-07-14T11:03:42.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:03:42.649+0000 7fa81a53e700 1 -- 172.21.15.40:0/2034251137 wait complete. 2023-07-14T11:03:42.653 INFO:teuthology.orchestra.run.smithi040.stderr:adjusted tunables profile to default 2023-07-14T11:03:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:42 smithi040 ceph-mon[97954]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:43.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:42 smithi040 ceph-mon[97954]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:43.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:42 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-07-14T11:03:43.110 INFO:tasks.cephadm:Adding mon.a on smithi040 2023-07-14T11:03:43.110 INFO:tasks.cephadm:Adding mon.b on smithi045 2023-07-14T11:03:43.110 INFO:tasks.cephadm:Adding mon.c on smithi130 2023-07-14T11:03:43.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch apply mon '3;smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c' 2023-07-14T11:03:43.708 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:43 smithi040 ceph-mon[97954]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:43 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-07-14T11:03:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:43 smithi040 ceph-mon[97954]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:44.855 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 -- 172.21.15.130:0/2562643519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 msgr2=0x7f82f41048c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:44.855 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 --2- 172.21.15.130:0/2562643519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f41048c0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f82e4009910 tx=0x7f82e4009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:44.855 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 -- 172.21.15.130:0/2562643519 shutdown_connections 2023-07-14T11:03:44.855 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 --2- 172.21.15.130:0/2562643519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f41048c0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 -- 172.21.15.130:0/2562643519 >> 172.21.15.130:0/2562643519 conn(0x7f82f40fa010 msgr2=0x7f82f40fc420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 -- 172.21.15.130:0/2562643519 shutdown_connections 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.850+0000 7f82f9f29700 1 -- 172.21.15.130:0/2562643519 wait complete. 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f9f29700 1 Processor -- start 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f9f29700 1 -- start start 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f9f29700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:44.856 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f9f29700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82f419ae90 con 0x7f82f41044f0 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.130:48194/0 (socket says 172.21.15.130:48194) 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.851+0000 7f82f37fe700 1 -- 172.21.15.130:0/3769528733 learned_addr learned my addr 172.21.15.130:0/3769528733 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f37fe700 1 -- 172.21.15.130:0/3769528733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82e40095f0 con 0x7f82f41044f0 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f37fe700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f82e4004f40 tx=0x7f82e4005020 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f82e401d070 con 0x7f82f41044f0 2023-07-14T11:03:44.857 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f82f419b090 con 0x7f82f41044f0 2023-07-14T11:03:44.858 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f82e4004370 con 0x7f82f41044f0 2023-07-14T11:03:44.858 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f82f419b4b0 con 0x7f82f41044f0 2023-07-14T11:03:44.858 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.852+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7f82e40039a0 con 0x7f82f41044f0 2023-07-14T11:03:44.859 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.853+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f82f4061b40 con 0x7f82f41044f0 2023-07-14T11:03:44.859 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.853+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f82e4003b00 con 0x7f82f41044f0 2023-07-14T11:03:44.859 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.853+0000 7f82f0ff9700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 0x7f82dc03ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:44.859 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.853+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f82e404c5a0 con 0x7f82f41044f0 2023-07-14T11:03:44.859 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.854+0000 7f82f2ffd700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 0x7f82dc03ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:44.860 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.854+0000 7f82f2ffd700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 0x7f82dc03ac60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f82e8006fd0 tx=0x7f82e8006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:44.861 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:44.859+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f82e400f460 con 0x7f82f41044f0 2023-07-14T11:03:45.055 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.052+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}) v1 -- 0x7f82f41022c0 con 0x7f82dc0387b0 2023-07-14T11:03:45.063 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.060+0000 7f82f0ff9700 1 -- 172.21.15.130:0/3769528733 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f82f41022c0 con 0x7f82dc0387b0 2023-07-14T11:03:45.063 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mon update... 2023-07-14T11:03:45.064 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 msgr2=0x7f82dc03ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:45.064 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 0x7f82dc03ac60 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f82e8006fd0 tx=0x7f82e8006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 msgr2=0x7f82f419a950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f82e4004f40 tx=0x7f82e4005020 comp rx=0 tx=0).stop 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 shutdown_connections 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f82dc0387b0 0x7f82dc03ac60 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 --2- 172.21.15.130:0/3769528733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82f41044f0 0x7f82f419a950 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 >> 172.21.15.130:0/3769528733 conn(0x7f82f40fa010 msgr2=0x7f82f4106850 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 shutdown_connections 2023-07-14T11:03:45.066 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:45.061+0000 7f82f9f29700 1 -- 172.21.15.130:0/3769528733 wait complete. 2023-07-14T11:03:45.472 DEBUG:teuthology.orchestra.run.smithi045:mon.b> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.b.service 2023-07-14T11:03:45.476 DEBUG:teuthology.orchestra.run.smithi130:mon.c> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.c.service 2023-07-14T11:03:45.479 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-07-14T11:03:45.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph mon dump -f json 2023-07-14T11:03:45.507 INFO:journalctl@ceph.mon.c.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:03:45.511 INFO:journalctl@ceph.mon.b.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:03:45.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:45.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:45.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:46.431 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:46 smithi040 ceph-mon[97954]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:46 smithi040 ceph-mon[97954]: Saving service mon spec with placement smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c;count:3 2023-07-14T11:03:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:46 smithi040 ceph-mon[97954]: Deploying daemon mon.c on smithi130 2023-07-14T11:03:47.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:47 smithi040 ceph-mon[97954]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.073 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- 172.21.15.130:0/862694346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 msgr2=0x7fdafc100be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:48.073 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 --2- 172.21.15.130:0/862694346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc100be0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fdaf0009910 tx=0x7fdaf0009c20 comp rx=0 tx=0).stop 2023-07-14T11:03:48.073 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- 172.21.15.130:0/862694346 shutdown_connections 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 --2- 172.21.15.130:0/862694346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc100be0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- 172.21.15.130:0/862694346 >> 172.21.15.130:0/862694346 conn(0x7fdafc0fa010 msgr2=0x7fdafc0fc420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- 172.21.15.130:0/862694346 shutdown_connections 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- 172.21.15.130:0/862694346 wait complete. 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 Processor -- start 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- start start 2023-07-14T11:03:48.074 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:48.075 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.070+0000 7fdb02faf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdafc19ade0 con 0x7fdafc100810 2023-07-14T11:03:48.076 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.071+0000 7fdb00d4b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:48.076 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.071+0000 7fdb00d4b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.130:48200/0 (socket says 172.21.15.130:48200) 2023-07-14T11:03:48.077 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.071+0000 7fdb00d4b700 1 -- 172.21.15.130:0/1522802682 learned_addr learned my addr 172.21.15.130:0/1522802682 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:03:48.077 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.071+0000 7fdb00d4b700 1 -- 172.21.15.130:0/1522802682 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdaf00095f0 con 0x7fdafc100810 2023-07-14T11:03:48.077 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.071+0000 7fdb00d4b700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fdaf0005850 tx=0x7fdaf0004ff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:48.077 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.072+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdaf001d070 con 0x7fdafc100810 2023-07-14T11:03:48.077 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.072+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdafc19afe0 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.072+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdaf0004370 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.072+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 205+0+0 (secure 0 0 0) 0x7fdaf0003ab0 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.072+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdafc19b400 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.073+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdafc061b40 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.073+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fdaf000b6f0 con 0x7fdafc100810 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.073+0000 7fdaf9ffb700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 0x7fdaec03ac60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:48.078 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.073+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fdaf004c560 con 0x7fdafc100810 2023-07-14T11:03:48.079 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.074+0000 7fdafbfff700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 0x7fdaec03ac60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:48.079 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.074+0000 7fdafbfff700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 0x7fdaec03ac60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fdae8006fd0 tx=0x7fdae8006e40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:48.081 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.079+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdaf0005c00 con 0x7fdafc100810 2023-07-14T11:03:48.369 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.365+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fdafc195e30 con 0x7fdafc100810 2023-07-14T11:03:48.369 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.366+0000 7fdaf9ffb700 1 -- 172.21.15.130:0/1522802682 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+736 (secure 0 0 0) 0x7fdaf0026030 con 0x7fdafc100810 2023-07-14T11:03:48.370 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:03:48.370 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","modified":"2023-07-14T11:02:05.545560Z","created":"2023-07-14T11:02:05.545560Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-07-14T11:03:48.370 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 msgr2=0x7fdaec03ac60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:48.370 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 0x7fdaec03ac60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fdae8006fd0 tx=0x7fdae8006e40 comp rx=0 tx=0).stop 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 msgr2=0x7fdafc19a8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fdaf0005850 tx=0x7fdaf0004ff0 comp rx=0 tx=0).stop 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 shutdown_connections 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdaec0387b0 0x7fdaec03ac60 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 --2- 172.21.15.130:0/1522802682 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdafc100810 0x7fdafc19a8a0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 >> 172.21.15.130:0/1522802682 conn(0x7fdafc0fa010 msgr2=0x7fdafc103e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:48.371 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 shutdown_connections 2023-07-14T11:03:48.372 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:48.368+0000 7fdb02faf700 1 -- 172.21.15.130:0/1522802682 wait complete. 2023-07-14T11:03:48.372 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 1 2023-07-14T11:03:48.397 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 systemd[1]: Starting Ceph mon.c for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:03:48.650 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 podman[129341]: 2023-07-14T11:03:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:48 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/1522802682' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:48.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: set uid:gid to 167:167 (ceph:ceph) 2023-07-14T11:03:48.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev), process ceph-mon, pid 7 2023-07-14T11:03:48.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pidfile_write: ignore empty --pid-file 2023-07-14T11:03:48.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: load: jerasure load: lrc 2023-07-14T11:03:48.932 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: RocksDB version: 7.9.2 2023-07-14T11:03:48.933 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Git sha 0 2023-07-14T11:03:48.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Compile date 2023-07-13 18:29:45 2023-07-14T11:03:48.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: DB SUMMARY 2023-07-14T11:03:48.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: DB Session ID: 2NU3WWZSRDI0FXQTD33B 2023-07-14T11:03:48.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: CURRENT file: CURRENT 2023-07-14T11:03:48.934 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: IDENTITY file: IDENTITY 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.error_if_exists: 0 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.create_if_missing: 0 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.paranoid_checks: 1 2023-07-14T11:03:48.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.flush_verify_memtable_count: 1 2023-07-14T11:03:48.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-07-14T11:03:48.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-07-14T11:03:48.936 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.env: 0x55fa233cd720 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.fs: PosixFileSystem 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.info_log: 0x55fa25b43280 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_file_opening_threads: 16 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.statistics: (nil) 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.use_fsync: 0 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_log_file_size: 0 2023-07-14T11:03:48.937 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.log_file_time_to_roll: 0 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.keep_log_file_num: 1000 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.recycle_log_file_num: 0 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_fallocate: 1 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_mmap_reads: 0 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_mmap_writes: 0 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.use_direct_reads: 0 2023-07-14T11:03:48.938 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-07-14T11:03:48.939 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.create_missing_column_families: 0 2023-07-14T11:03:48.939 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.db_log_dir: 2023-07-14T11:03:48.939 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.wal_dir: 2023-07-14T11:03:48.939 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.table_cache_numshardbits: 6 2023-07-14T11:03:48.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.WAL_ttl_seconds: 0 2023-07-14T11:03:48.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.WAL_size_limit_MB: 0 2023-07-14T11:03:48.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-07-14T11:03:48.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-07-14T11:03:48.941 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.is_fd_close_on_exec: 1 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.advise_random_on_open: 1 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.db_write_buffer_size: 0 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.write_buffer_manager: 0x55fa25a643c0 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.use_adaptive_mutex: 0 2023-07-14T11:03:48.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.rate_limiter: (nil) 2023-07-14T11:03:48.943 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-07-14T11:03:48.943 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.wal_recovery_mode: 2 2023-07-14T11:03:48.943 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enable_thread_tracking: 0 2023-07-14T11:03:48.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enable_pipelined_write: 0 2023-07-14T11:03:48.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.unordered_write: 0 2023-07-14T11:03:48.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-07-14T11:03:48.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-07-14T11:03:48.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-07-14T11:03:48.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-07-14T11:03:48.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.row_cache: None 2023-07-14T11:03:48.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.wal_filter: None 2023-07-14T11:03:48.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-07-14T11:03:48.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_ingest_behind: 0 2023-07-14T11:03:48.946 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.two_write_queues: 0 2023-07-14T11:03:48.947 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.manual_wal_flush: 0 2023-07-14T11:03:48.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.wal_compression: 0 2023-07-14T11:03:48.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.atomic_flush: 0 2023-07-14T11:03:48.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.persist_stats_to_disk: 0 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.write_dbid_to_manifest: 0 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.log_readahead_size: 0 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.best_efforts_recovery: 0 2023-07-14T11:03:48.949 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.allow_data_in_errors: 0 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.db_host_id: __hostname__ 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enforce_single_del_contracts: true 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_background_jobs: 2 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_background_compactions: -1 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_subcompactions: 1 2023-07-14T11:03:48.950 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.delayed_write_rate : 16777216 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_total_wal_size: 0 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.stats_dump_period_sec: 600 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.stats_persist_period_sec: 600 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-07-14T11:03:48.951 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_open_files: -1 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bytes_per_sync: 0 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.wal_bytes_per_sync: 0 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.strict_bytes_per_sync: 0 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_readahead_size: 0 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_background_flushes: -1 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Compression algorithms supported: 2023-07-14T11:03:48.952 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kZSTD supported: 0 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kXpressCompression supported: 0 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kLZ4HCCompression supported: 1 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kZlibCompression supported: 1 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kSnappyCompression supported: 1 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kLZ4Compression supported: 1 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: kBZip2Compression supported: 0 2023-07-14T11:03:48.953 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Fast CRC32 supported: Supported on x86 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: DMutex implementation: pthread_mutex_t 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.merge_operator: 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_filter: None 2023-07-14T11:03:48.954 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_filter_factory: None 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.sst_partitioner_factory: None 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.memtable_factory: SkipListFactory 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.table_factory: BlockBasedTable 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fa25b43380) 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-07-14T11:03:48.955 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 4 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2023-07-14T11:03:48.956 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x55fa25b031f0 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2023-07-14T11:03:48.957 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2023-07-14T11:03:48.958 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: bloomfilter 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 5 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_auto_readahead_size: 262144 2023-07-14T11:03:48.959 INFO:journalctl@ceph.mon.c.smithi130.stdout: prepopulate_block_cache: 0 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout: initial_auto_readahead_size: 8192 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.write_buffer_size: 33554432 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_write_buffer_number: 2 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression: NoCompression 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression: Disabled 2023-07-14T11:03:48.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.prefix_extractor: nullptr 2023-07-14T11:03:48.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-07-14T11:03:48.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.num_levels: 7 2023-07-14T11:03:48.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-07-14T11:03:48.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-07-14T11:03:48.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-07-14T11:03:48.962 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-07-14T11:03:48.962 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-07-14T11:03:48.962 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-07-14T11:03:48.962 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-07-14T11:03:48.963 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:03:48.963 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-07-14T11:03:48.963 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-07-14T11:03:48.963 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:03:48.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:03:48.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.window_bits: -14 2023-07-14T11:03:48.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.level: 32767 2023-07-14T11:03:48.965 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.strategy: 0 2023-07-14T11:03:48.965 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-07-14T11:03:48.965 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:03:48.965 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:03:48.965 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.enabled: false 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.target_file_size_base: 67108864 2023-07-14T11:03:48.966 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.target_file_size_multiplier: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-07-14T11:03:48.967 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.arena_block_size: 1048576 2023-07-14T11:03:48.968 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.disable_auto_compactions: 0 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-07-14T11:03:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.table_properties_collectors: 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.inplace_update_support: 0 2023-07-14T11:03:48.970 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.inplace_update_num_locks: 10000 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.memtable_huge_page_size: 0 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.bloom_locality: 0 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.max_successive_merges: 0 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.optimize_filters_for_hits: 0 2023-07-14T11:03:48.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.paranoid_file_checks: 0 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.force_consistency_checks: 1 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.report_bg_io_stats: 0 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.ttl: 2592000 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.periodic_compaction_seconds: 0 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-07-14T11:03:48.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enable_blob_files: false 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.min_blob_size: 0 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_file_size: 268435456 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_compression_type: NoCompression 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.enable_blob_garbage_collection: false 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-07-14T11:03:48.973 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.blob_file_starting_level: 0 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 29c94976-8f92-4ac1-b5a2-1c59c5df0c0c 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332628819411, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-07-14T11:03:48.974 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332628819812, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1689332628, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "29c94976-8f92-4ac1-b5a2-1c59c5df0c0c", "db_session_id": "2NU3WWZSRDI0FXQTD33B", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332628819879, "job": 1, "event": "recovery_finished"} 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fa25b9c000 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: DB pointer 0x55fa25b88000 2023-07-14T11:03:48.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:03:48.976 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:03:48.977 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-07-14T11:03:48.978 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-07-14T11:03:48.979 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache BinnedLRUCache@0x55fa25b031f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2023-07-14T11:03:48.980 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: using public_addr v2:172.21.15.130:0/0 -> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: starting mon.c rank -1 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-c fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:48.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(???) e0 preinit fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:48.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).mds e1 new map 2023-07-14T11:03:48.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).mds e1 print_map 2023-07-14T11:03:48.982 INFO:journalctl@ceph.mon.c.smithi130.stdout: e1 2023-07-14T11:03:48.983 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-07-14T11:03:48.983 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-07-14T11:03:48.983 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: -1 2023-07-14T11:03:48.983 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2023-07-14T11:03:48.984 INFO:journalctl@ceph.mon.c.smithi130.stdout: No filesystems configured 2023-07-14T11:03:48.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-07-14T11:03:48.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-07-14T11:03:48.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-07-14T11:03:48.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-07-14T11:03:48.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-07-14T11:03:48.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-07-14T11:03:48.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-07-14T11:03:48.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:48.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:48.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:48.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:48.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Deploying cephadm binary to smithi130 2023-07-14T11:03:48.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Added host smithi130 2023-07-14T11:03:48.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:48.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.991 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:03:48.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:48.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Saving service mon spec with placement smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c;count:3 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: Deploying daemon mon.c on smithi130 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/1522802682' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:48.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expand map: {default=false} 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta from 'false' to 'false' 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expanded map: {default=false} 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expand map: {default=info} 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta from 'info' to 'info' 2023-07-14T11:03:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expanded map: {default=info} 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expand map: {default=daemon} 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta from 'daemon' to 'daemon' 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expanded map: {default=daemon} 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expand map: {default=debug} 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta from 'debug' to 'debug' 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 ceph-mon[129405]: expand_channel_meta expanded map: {default=debug} 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 bash[129341]: b97a5855a52ddfa21fbf672714e2e77fb9022498c5b303150ed5650c40085fae 2023-07-14T11:03:48.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:48 smithi130 systemd[1]: Started Ceph mon.c for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:03:49.687 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-07-14T11:03:49.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph mon dump -f json 2023-07-14T11:03:50.245 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:03:54.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: mon.a calling monitor election 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: mon.c calling monitor election 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:03:54.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: monmap e2: 2 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: fsmap 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: mgrmap e13: x(active, since 47s) 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: overall HEALTH_OK 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:54.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:54.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: mon.a calling monitor election 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: mon.c calling monitor election 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: monmap e2: 2 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: fsmap 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: mgrmap e13: x(active, since 47s) 2023-07-14T11:03:54.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: overall HEALTH_OK 2023-07-14T11:03:54.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:54.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:54.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:54.426 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.422+0000 7f3008f73700 1 -- 172.21.15.130:0/3756296432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 msgr2=0x7f2ff0005a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:54.426 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 --2- 172.21.15.130:0/3756296432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f2ff0005a40 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f30040fe540 tx=0x7f2ff400feb0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.426 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 -- 172.21.15.130:0/3756296432 shutdown_connections 2023-07-14T11:03:54.427 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 --2- 172.21.15.130:0/3756296432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f2ff0005a40 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.427 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 -- 172.21.15.130:0/3756296432 >> 172.21.15.130:0/3756296432 conn(0x7f30040fa130 msgr2=0x7f30040fc540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:54.427 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 -- 172.21.15.130:0/3756296432 shutdown_connections 2023-07-14T11:03:54.427 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.423+0000 7f3008f73700 1 -- 172.21.15.130:0/3756296432 wait complete. 2023-07-14T11:03:54.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 Processor -- start 2023-07-14T11:03:54.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 -- start start 2023-07-14T11:03:54.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f300419a8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:54.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:54.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f300419b330 con 0x7f2ff00035c0 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3008f73700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f300419b4a0 con 0x7f300419ae00 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3001d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3001d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39272/0 (socket says 172.21.15.130:39272) 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.424+0000 7f3001d9b700 1 -- 172.21.15.130:0/1062140600 learned_addr learned my addr 172.21.15.130:0/1062140600 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f3001d9b700 1 -- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 msgr2=0x7f3004194940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 13 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f3001d9b700 1 -- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 msgr2=0x7f3004194940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-07-14T11:03:54.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f3001d9b700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-07-14T11:03:54.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f3001d9b700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:03:54.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f300259c700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f300419a8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:54.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f300259c700 1 -- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 msgr2=0x7f3004194940 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:03:54.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f300259c700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.425+0000 7f300259c700 1 -- 172.21.15.130:0/1062140600 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ff400f900 con 0x7f2ff00035c0 2023-07-14T11:03:54.431 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f300259c700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f300419a8c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f30040fe540 tx=0x7f2ff40047a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7f2ff4013b70 con 0x7f2ff00035c0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2ff40044f0 con 0x7f2ff00035c0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3004194f40 con 0x7f2ff00035c0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 309+0+0 (secure 0 0 0) 0x7f2ff4003c80 con 0x7f2ff00035c0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.426+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3004195510 con 0x7f2ff00035c0 2023-07-14T11:03:54.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.427+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f2ff4011470 con 0x7f2ff00035c0 2023-07-14T11:03:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.427+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f300418f290 con 0x7f2ff00035c0 2023-07-14T11:03:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.427+0000 7f2ffb7fe700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 0x7f2fe403aae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:03:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.427+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f2ff404dd10 con 0x7f2ff00035c0 2023-07-14T11:03:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.428+0000 7f3001d9b700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 0x7f2fe403aae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:03:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.428+0000 7f3001d9b700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 0x7f2fe403aae0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2fec009180 tx=0x7f2fec008040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:03:54.435 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.433+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f2ff40109c0 con 0x7f2ff00035c0 2023-07-14T11:03:54.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.695+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3004195780 con 0x7f2ff00035c0 2023-07-14T11:03:54.699 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.697+0000 7f2ffb7fe700 1 -- 172.21.15.130:0/1062140600 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) v1 ==== 95+0+1006 (secure 0 0 0) 0x7f2ff40103d0 con 0x7f2ff00035c0 2023-07-14T11:03:54.699 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:03:54.699 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":2,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","modified":"2023-07-14T11:03:48.848315Z","created":"2023-07-14T11:02:05.545560Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-07-14T11:03:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 msgr2=0x7f2fe403aae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 0x7f2fe403aae0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2fec009180 tx=0x7f2fec008040 comp rx=0 tx=0).stop 2023-07-14T11:03:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 msgr2=0x7f300419a8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:03:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f300419a8c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f30040fe540 tx=0x7f2ff40047a0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 shutdown_connections 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2fe4038630 0x7f2fe403aae0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ff00035c0 0x7f300419a8c0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 --2- 172.21.15.130:0/1062140600 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300419ae00 0x7f3004194940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 >> 172.21.15.130:0/1062140600 conn(0x7f30040fa130 msgr2=0x7f3004068820 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 shutdown_connections 2023-07-14T11:03:54.702 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:03:54.698+0000 7f3008f73700 1 -- 172.21.15.130:0/1062140600 wait complete. 2023-07-14T11:03:54.703 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 2 2023-07-14T11:03:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:54 smithi130 ceph-mon[129405]: Deploying daemon mon.b on smithi045 2023-07-14T11:03:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:54 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/1062140600' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:54 smithi040 ceph-mon[97954]: Deploying daemon mon.b on smithi045 2023-07-14T11:03:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:54 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/1062140600' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:56.098 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-07-14T11:03:56.098 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph mon dump -f json 2023-07-14T11:03:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:03:55 smithi130 ceph-mon[129405]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:56.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:03:55 smithi040 ceph-mon[97954]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:56.661 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:03:56.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:56 smithi045 systemd[1]: Starting Ceph mon.b for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:03:57.162 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:56 smithi045 podman[128751]: 2023-07-14T11:03:57.162 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: set uid:gid to 167:167 (ceph:ceph) 2023-07-14T11:03:57.163 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev), process ceph-mon, pid 7 2023-07-14T11:03:57.163 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pidfile_write: ignore empty --pid-file 2023-07-14T11:03:57.163 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: load: jerasure load: lrc 2023-07-14T11:03:57.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: RocksDB version: 7.9.2 2023-07-14T11:03:57.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Git sha 0 2023-07-14T11:03:57.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Compile date 2023-07-13 18:29:45 2023-07-14T11:03:57.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: DB SUMMARY 2023-07-14T11:03:57.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: DB Session ID: AJQIL69EM6Z1GESCAMBI 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: CURRENT file: CURRENT 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: IDENTITY file: IDENTITY 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.error_if_exists: 0 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.create_if_missing: 0 2023-07-14T11:03:57.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.paranoid_checks: 1 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.flush_verify_memtable_count: 1 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.env: 0x55ff0a32c720 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.fs: PosixFileSystem 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.info_log: 0x55ff0c7cd280 2023-07-14T11:03:57.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_file_opening_threads: 16 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.statistics: (nil) 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.use_fsync: 0 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_log_file_size: 0 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.log_file_time_to_roll: 0 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.keep_log_file_num: 1000 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.recycle_log_file_num: 0 2023-07-14T11:03:57.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_fallocate: 1 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_mmap_reads: 0 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_mmap_writes: 0 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.use_direct_reads: 0 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.create_missing_column_families: 0 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.db_log_dir: 2023-07-14T11:03:57.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.wal_dir: 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.table_cache_numshardbits: 6 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.WAL_ttl_seconds: 0 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.WAL_size_limit_MB: 0 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.is_fd_close_on_exec: 1 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.advise_random_on_open: 1 2023-07-14T11:03:57.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.db_write_buffer_size: 0 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.write_buffer_manager: 0x55ff0c6ee3c0 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.use_adaptive_mutex: 0 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.rate_limiter: (nil) 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.wal_recovery_mode: 2 2023-07-14T11:03:57.571 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enable_thread_tracking: 0 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enable_pipelined_write: 0 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.unordered_write: 0 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.row_cache: None 2023-07-14T11:03:57.572 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 bash[128751]: dac75a1cd6c1795bd19415fcd6732b203802af4d51797377a5caf1fd72947a93 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.wal_filter: None 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_ingest_behind: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.two_write_queues: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.manual_wal_flush: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.wal_compression: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.atomic_flush: 0 2023-07-14T11:03:57.573 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.persist_stats_to_disk: 0 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.write_dbid_to_manifest: 0 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.log_readahead_size: 0 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.best_efforts_recovery: 0 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-07-14T11:03:57.574 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.allow_data_in_errors: 0 2023-07-14T11:03:57.575 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.db_host_id: __hostname__ 2023-07-14T11:03:57.575 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enforce_single_del_contracts: true 2023-07-14T11:03:57.575 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_background_jobs: 2 2023-07-14T11:03:57.575 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_background_compactions: -1 2023-07-14T11:03:57.576 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_subcompactions: 1 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.delayed_write_rate : 16777216 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_total_wal_size: 0 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.stats_dump_period_sec: 600 2023-07-14T11:03:57.577 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.stats_persist_period_sec: 600 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_open_files: -1 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bytes_per_sync: 0 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.wal_bytes_per_sync: 0 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.strict_bytes_per_sync: 0 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_readahead_size: 0 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_background_flushes: -1 2023-07-14T11:03:57.578 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Compression algorithms supported: 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kZSTD supported: 0 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kXpressCompression supported: 0 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kLZ4HCCompression supported: 1 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kZlibCompression supported: 1 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kSnappyCompression supported: 1 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kLZ4Compression supported: 1 2023-07-14T11:03:57.579 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: kBZip2Compression supported: 0 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Fast CRC32 supported: Supported on x86 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: DMutex implementation: pthread_mutex_t 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.merge_operator: 2023-07-14T11:03:57.580 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_filter: None 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_filter_factory: None 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.sst_partitioner_factory: None 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.memtable_factory: SkipListFactory 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.table_factory: BlockBasedTable 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ff0c7cd380) 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout: cache_index_and_filter_blocks: 1 2023-07-14T11:03:57.581 INFO:journalctl@ceph.mon.b.smithi045.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: pin_top_level_index_and_filter: 1 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_type: 0 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: data_block_index_type: 0 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_shortening: 1 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: data_block_hash_table_util_ratio: 0.750000 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: checksum: 4 2023-07-14T11:03:57.582 INFO:journalctl@ceph.mon.b.smithi045.stdout: no_block_cache: 0 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache: 0x55ff0c78d1f0 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_name: BinnedLRUCache 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_options: 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: capacity : 536870912 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: num_shard_bits : 4 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: strict_capacity_limit : 0 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: high_pri_pool_ratio: 0.000 2023-07-14T11:03:57.583 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_cache_compressed: (nil) 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: persistent_cache: (nil) 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_size: 4096 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_size_deviation: 10 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_restart_interval: 16 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: index_block_restart_interval: 1 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: metadata_block_size: 4096 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: partition_filters: 0 2023-07-14T11:03:57.584 INFO:journalctl@ceph.mon.b.smithi045.stdout: use_delta_encoding: 1 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: filter_policy: bloomfilter 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: whole_key_filtering: 1 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: verify_compression: 0 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: read_amp_bytes_per_bit: 0 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: format_version: 5 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: enable_index_compression: 1 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: block_align: 0 2023-07-14T11:03:57.585 INFO:journalctl@ceph.mon.b.smithi045.stdout: max_auto_readahead_size: 262144 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout: prepopulate_block_cache: 0 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout: initial_auto_readahead_size: 8192 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout: num_file_reads_for_auto_readahead: 2 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.write_buffer_size: 33554432 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_write_buffer_number: 2 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression: NoCompression 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression: Disabled 2023-07-14T11:03:57.586 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.prefix_extractor: nullptr 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.num_levels: 7 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-07-14T11:03:57.587 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:03:57.588 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.window_bits: -14 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.level: 32767 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.strategy: 0 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.enabled: false 2023-07-14T11:03:57.589 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.target_file_size_base: 67108864 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.target_file_size_multiplier: 1 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-07-14T11:03:57.590 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-07-14T11:03:57.591 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.arena_block_size: 1048576 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.disable_auto_compactions: 0 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-07-14T11:03:57.592 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-07-14T11:03:57.593 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.table_properties_collectors: 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.inplace_update_support: 0 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.inplace_update_num_locks: 10000 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.memtable_huge_page_size: 0 2023-07-14T11:03:57.594 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.bloom_locality: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.max_successive_merges: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.optimize_filters_for_hits: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.paranoid_file_checks: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.force_consistency_checks: 1 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.report_bg_io_stats: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.ttl: 2592000 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.periodic_compaction_seconds: 0 2023-07-14T11:03:57.595 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enable_blob_files: false 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.min_blob_size: 0 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_file_size: 268435456 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_compression_type: NoCompression 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.enable_blob_garbage_collection: false 2023-07-14T11:03:57.596 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.blob_file_starting_level: 0 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: df282cd2-fba1-4afc-9c58-1e17d0c5c775 2023-07-14T11:03:57.597 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332637162547, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332637162953, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1689332637, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "df282cd2-fba1-4afc-9c58-1e17d0c5c775", "db_session_id": "AJQIL69EM6Z1GESCAMBI", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: EVENT_LOG_v1 {"time_micros": 1689332637163023, "job": 1, "event": "recovery_finished"} 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ff0c826000 2023-07-14T11:03:57.598 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: DB pointer 0x55ff0c812000 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** DB Stats ** 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:57.599 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** Compaction Stats [default] ** 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-07-14T11:03:57.600 INFO:journalctl@ceph.mon.b.smithi045.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** Compaction Stats [default] ** 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-07-14T11:03:57.601 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-07-14T11:03:57.602 INFO:journalctl@ceph.mon.b.smithi045.stdout: AddFile(Keys): cumulative 0, interval 0 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: Cumulative compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: Interval compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: Block cache BinnedLRUCache@0x55ff0c78d1f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 7e-06 secs_since: 0 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.603 INFO:journalctl@ceph.mon.b.smithi045.stdout: ** File Read Latency Histogram By Level [default] ** 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: using public_addr v2:172.21.15.45:0/0 -> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: starting mon.b rank -1 at public addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] at bind addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(???) e0 preinit fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 systemd[1]: Started Ceph mon.b for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).mds e1 new map 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).mds e1 print_map 2023-07-14T11:03:57.604 INFO:journalctl@ceph.mon.b.smithi045.stdout: e1 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout: legacy client fscid: -1 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout: 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout: No filesystems configured 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-07-14T11:03:57.605 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:57.606 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:57.607 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Deploying cephadm binary to smithi130 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Added host smithi130 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.610 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.611 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2034251137' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:57.612 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:57.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Saving service mon spec with placement smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c;count:3 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Deploying daemon mon.c on smithi130 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/1522802682' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:03:57.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.a calling monitor election 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.c calling monitor election 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.615 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: monmap e2: 2 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: fsmap 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mgrmap e13: x(active, since 47s) 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: overall HEALTH_OK 2023-07-14T11:03:57.616 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: Deploying daemon mon.b on smithi045 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/1062140600' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:03:57.617 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:03:57.618 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-07-14T11:03:57.618 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-07-14T11:03:57.618 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expand map: {default=false} 2023-07-14T11:03:57.618 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta from 'false' to 'false' 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expanded map: {default=false} 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expand map: {default=info} 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta from 'info' to 'info' 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expanded map: {default=info} 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expand map: {default=daemon} 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta from 'daemon' to 'daemon' 2023-07-14T11:03:57.619 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expanded map: {default=daemon} 2023-07-14T11:03:57.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expand map: {default=debug} 2023-07-14T11:03:57.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta from 'debug' to 'debug' 2023-07-14T11:03:57.620 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:03:57 smithi045 ceph-mon[128791]: expand_channel_meta expanded map: {default=debug} 2023-07-14T11:04:02.291 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 -- 172.21.15.130:0/3540654406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898102df0 msgr2=0x7f0884007820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:02.291 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 --2- 172.21.15.130:0/3540654406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898102df0 0x7f0884007820 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f0888005fa0 tx=0x7f088800cf80 comp rx=0 tx=0).stop 2023-07-14T11:04:02.291 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 -- 172.21.15.130:0/3540654406 shutdown_connections 2023-07-14T11:04:02.292 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 --2- 172.21.15.130:0/3540654406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898102df0 0x7f0884007820 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.292 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 -- 172.21.15.130:0/3540654406 >> 172.21.15.130:0/3540654406 conn(0x7f0898073d70 msgr2=0x7f0898074170 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:02.292 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 -- 172.21.15.130:0/3540654406 shutdown_connections 2023-07-14T11:04:02.292 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.288+0000 7f089e0be700 1 -- 172.21.15.130:0/3540654406 wait complete. 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.289+0000 7f089e0be700 1 Processor -- start 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.289+0000 7f089e0be700 1 -- start start 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 0x7f08981928a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0898199370 con 0x7f0898198d60 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089819f640 con 0x7f0898192de0 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f089e0be700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f089819f780 con 0x7f0898102df0 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39298/0 (socket says 172.21.15.130:39298) 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 -- 172.21.15.130:0/1392470229 learned_addr learned my addr 172.21.15.130:0/1392470229 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 msgr2=0x7f0898198820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 12 2023-07-14T11:04:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 msgr2=0x7f0898198820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 0x7f08981928a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f08977fe700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0897fff700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0897fff700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 msgr2=0x7f0898193280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 14 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0897fff700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 msgr2=0x7f0898193280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0897fff700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 2023-07-14T11:04:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0897fff700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 msgr2=0x7f0898198820 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 msgr2=0x7f0898193280 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.290+0000 7f0896ffd700 1 -- 172.21.15.130:0/1392470229 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f088800c980 con 0x7f0898198d60 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f0896ffd700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 0x7f08981928a0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f088000b550 tx=0x7f088000b910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0880004230 con 0x7f0898198d60 2023-07-14T11:04:02.297 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0880004390 con 0x7f0898198d60 2023-07-14T11:04:02.298 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f089819fa60 con 0x7f0898198d60 2023-07-14T11:04:02.298 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0880011af0 con 0x7f0898198d60 2023-07-14T11:04:02.298 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.291+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f089819ff60 con 0x7f0898198d60 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.292+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f0880011c50 con 0x7f0898198d60 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.292+0000 7f0894ff9700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 0x7f087803b0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.292+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0898061b40 con 0x7f0898198d60 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.292+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f088004adc0 con 0x7f0898198d60 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.293+0000 7f08977fe700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 0x7f087803b0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:02.299 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.293+0000 7f08977fe700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 0x7f087803b0e0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f088800e7e0 tx=0x7f0888019470 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:02.300 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.298+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f088001a400 con 0x7f0898198d60 2023-07-14T11:04:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:04:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: mon.c calling monitor election 2023-07-14T11:04:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: mon.a calling monitor election 2023-07-14T11:04:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:02.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: fsmap 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: mgrmap e13: x(active, since 55s) 2023-07-14T11:04:02.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: mon.b (rank 2) addr [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] is down (out of quorum) 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.546 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:04:02.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: mon.c calling monitor election 2023-07-14T11:04:02.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: mon.a calling monitor election 2023-07-14T11:04:02.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:04:02.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:02.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: fsmap 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: mgrmap e13: x(active, since 55s) 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: mon.b (rank 2) addr [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] is down (out of quorum) 2023-07-14T11:04:02.550 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.551 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:02.576 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.573+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f08981a0310 con 0x7f0898198d60 2023-07-14T11:04:02.577 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.574+0000 7f0894ff9700 1 -- 172.21.15.130:0/1392470229 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 3 v3) v1 ==== 95+0+1270 (secure 0 0 0) 0x7f0880048020 con 0x7f0898198d60 2023-07-14T11:04:02.577 INFO:teuthology.orchestra.run.smithi130.stdout: 2023-07-14T11:04:02.577 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":3,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","modified":"2023-07-14T11:03:57.187263Z","created":"2023-07-14T11:02:05.545560Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-07-14T11:04:02.579 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 msgr2=0x7f087803b0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:02.579 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 0x7f087803b0e0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f088800e7e0 tx=0x7f0888019470 comp rx=0 tx=0).stop 2023-07-14T11:04:02.579 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 msgr2=0x7f08981928a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:02.579 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 0x7f08981928a0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f088000b550 tx=0x7f088000b910 comp rx=0 tx=0).stop 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 shutdown_connections 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0878038c30 0x7f087803b0e0 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898192de0 0x7f0898193280 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898102df0 0x7f0898198820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 --2- 172.21.15.130:0/1392470229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898198d60 0x7f08981928a0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 >> 172.21.15.130:0/1392470229 conn(0x7f0898073d70 msgr2=0x7f089818b910 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 shutdown_connections 2023-07-14T11:04:02.581 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:02.576+0000 7f089e0be700 1 -- 172.21.15.130:0/1392470229 wait complete. 2023-07-14T11:04:02.581 INFO:teuthology.orchestra.run.smithi130.stderr:dumped monmap epoch 3 2023-07-14T11:04:03.031 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-07-14T11:04:03.031 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph config generate-minimal-conf 2023-07-14T11:04:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:03 smithi130 ceph-mon[129405]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:03 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/1392470229' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:04:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:03.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:03.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:03.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:03 smithi040 ceph-mon[97954]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:03.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:03 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/1392470229' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:04:03.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:03.775 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:04:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:04:04.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.c calling monitor election 2023-07-14T11:04:04.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.a calling monitor election 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.567 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: fsmap 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:04.568 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mgrmap e13: x(active, since 55s) 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.b (rank 2) addr [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] is down (out of quorum) 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:04.569 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:04.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:04.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:04.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/1392470229' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-07-14T11:04:04.570 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:05.262 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mon.b calling monitor election 2023-07-14T11:04:05.262 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mon.b calling monitor election 2023-07-14T11:04:05.262 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mon.c calling monitor election 2023-07-14T11:04:05.262 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mon.a calling monitor election 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: fsmap 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: mgrmap e13: x(active, since 57s) 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-07-14T11:04:05.263 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: Cluster is now healthy 2023-07-14T11:04:05.264 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: overall HEALTH_OK 2023-07-14T11:04:05.264 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.264 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.264 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.264 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mon.b calling monitor election 2023-07-14T11:04:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mon.b calling monitor election 2023-07-14T11:04:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mon.c calling monitor election 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mon.a calling monitor election 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: fsmap 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: mgrmap e13: x(active, since 57s) 2023-07-14T11:04:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: Cluster is now healthy 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: overall HEALTH_OK 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.b calling monitor election 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.b calling monitor election 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.c calling monitor election 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.a calling monitor election 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: monmap e3: 3 mons at {a=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],b=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],c=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: fsmap 2023-07-14T11:04:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: osdmap e4: 0 total, 0 up, 0 in 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: mgrmap e13: x(active, since 57s) 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: Cluster is now healthy 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: overall HEALTH_OK 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:05.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 -- 172.21.15.40:0/559414028 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0105630 msgr2=0x7f5ef0105a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:05.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/559414028 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0105630 0x7f5ef0105a00 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f5eec009910 tx=0x7f5eec009c20 comp rx=0 tx=0).stop 2023-07-14T11:04:05.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 -- 172.21.15.40:0/559414028 shutdown_connections 2023-07-14T11:04:05.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/559414028 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0105630 0x7f5ef0105a00 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 -- 172.21.15.40:0/559414028 >> 172.21.15.40:0/559414028 conn(0x7f5ef00fb130 msgr2=0x7f5ef00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:05.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.646+0000 7f5ef7c90700 1 -- 172.21.15.40:0/559414028 shutdown_connections 2023-07-14T11:04:05.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.647+0000 7f5ef7c90700 1 -- 172.21.15.40:0/559414028 wait complete. 2023-07-14T11:04:05.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 Processor -- start 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 -- start start 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ef0105630 0x7f5ef019a9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ef019af10 0x7f5ef01949c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ef019b4b0 con 0x7f5ef0194f00 2023-07-14T11:04:05.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ef01a1780 con 0x7f5ef019af10 2023-07-14T11:04:05.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.648+0000 7f5ef7c90700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ef01a18c0 con 0x7f5ef0105630 2023-07-14T11:04:05.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:05.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49996/0 (socket says 172.21.15.40:49996) 2023-07-14T11:04:05.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 -- 172.21.15.40:0/2339328517 learned_addr learned my addr 172.21.15.40:0/2339328517 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:04:05.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef5a2c700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ef0105630 0x7f5ef019a9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:05.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef522b700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ef019af10 0x7f5ef01949c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 -- 172.21.15.40:0/2339328517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ef0105630 msgr2=0x7f5ef019a9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ef0105630 0x7f5ef019a9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 -- 172.21.15.40:0/2339328517 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ef019af10 msgr2=0x7f5ef01949c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ef019af10 0x7f5ef01949c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 -- 172.21.15.40:0/2339328517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5eec0095f0 con 0x7f5ef0194f00 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.649+0000 7f5ef622d700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f5ee800ba50 tx=0x7f5ee800bd60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:05.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.650+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ee800c9b0 con 0x7f5ef0194f00 2023-07-14T11:04:05.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.650+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ef01a1ba0 con 0x7f5ef0194f00 2023-07-14T11:04:05.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.650+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ee8014440 con 0x7f5ef0194f00 2023-07-14T11:04:05.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.650+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ee8007590 con 0x7f5ef0194f00 2023-07-14T11:04:05.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.650+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ef0072740 con 0x7f5ef0194f00 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.651+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f5ee80077b0 con 0x7f5ef0194f00 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.651+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5ef0061b40 con 0x7f5ef0194f00 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.652+0000 7f5ee6ffd700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 0x7f5edc03f540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.652+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f5ee804ab40 con 0x7f5ef0194f00 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.652+0000 7f5ef5a2c700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 0x7f5edc03f540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:05.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.653+0000 7f5ef5a2c700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 0x7f5edc03f540 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f5eec009fd0 tx=0x7f5eec0058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:05.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.657+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5ee801b8b0 con 0x7f5ef0194f00 2023-07-14T11:04:05.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.876+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f5ef0072b40 con 0x7f5ef0194f00 2023-07-14T11:04:05.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.876+0000 7f5ee6ffd700 1 -- 172.21.15.40:0/2339328517 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v9) v1 ==== 76+0+273 (secure 0 0 0) 0x7f5ee8012340 con 0x7f5ef0194f00 2023-07-14T11:04:05.879 INFO:teuthology.orchestra.run.smithi040.stdout:# minimal ceph.conf for bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:04:05.879 INFO:teuthology.orchestra.run.smithi040.stdout:[global] 2023-07-14T11:04:05.879 INFO:teuthology.orchestra.run.smithi040.stdout: fsid = bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:04:05.880 INFO:teuthology.orchestra.run.smithi040.stdout: mon_host = [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] 2023-07-14T11:04:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 msgr2=0x7f5edc03f540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 0x7f5edc03f540 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f5eec009fd0 tx=0x7f5eec0058e0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 msgr2=0x7f5ef01953a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f5ee800ba50 tx=0x7f5ee800bd60 comp rx=0 tx=0).stop 2023-07-14T11:04:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 shutdown_connections 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5edc03d090 0x7f5edc03f540 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ef0194f00 0x7f5ef01953a0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ef0105630 0x7f5ef019a9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 --2- 172.21.15.40:0/2339328517 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ef019af10 0x7f5ef01949c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 >> 172.21.15.40:0/2339328517 conn(0x7f5ef00fb130 msgr2=0x7f5ef00fb500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 shutdown_connections 2023-07-14T11:04:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:05.879+0000 7f5ef7c90700 1 -- 172.21.15.40:0/2339328517 wait complete. 2023-07-14T11:04:06.195 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.195 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:06.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:06.197 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:06.197 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.197 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:05 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2339328517' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:05 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2339328517' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:06.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:05 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2339328517' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:06.879 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-07-14T11:04:06.880 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:04:06.880 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/ceph/ceph.conf 2023-07-14T11:04:06.956 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:04:06.956 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:04:07.056 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:06 smithi040 ceph-mon[97954]: Reconfiguring mon.a (unknown last config time)... 2023-07-14T11:04:07.057 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:06 smithi040 ceph-mon[97954]: Reconfiguring daemon mon.a on smithi040 2023-07-14T11:04:07.057 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.057 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.058 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.066 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:04:07.067 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.conf 2023-07-14T11:04:07.100 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:04:07.101 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:04:07.169 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:04:07.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2023-07-14T11:04:07.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:06 smithi130 ceph-mon[129405]: Reconfiguring mon.a (unknown last config time)... 2023-07-14T11:04:07.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:06 smithi130 ceph-mon[129405]: Reconfiguring daemon mon.a on smithi040 2023-07-14T11:04:07.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.203 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.207 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:04:07.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:04:07.278 INFO:tasks.cephadm:Adding mgr.x on smithi040 2023-07-14T11:04:07.279 INFO:tasks.cephadm:Adding mgr.y on smithi045 2023-07-14T11:04:07.279 INFO:tasks.cephadm:Adding mgr.z on smithi130 2023-07-14T11:04:07.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch apply mgr '3;smithi040=x;smithi045=y;smithi130=z' 2023-07-14T11:04:07.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:06 smithi045 ceph-mon[128791]: Reconfiguring mon.a (unknown last config time)... 2023-07-14T11:04:07.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:06 smithi045 ceph-mon[128791]: Reconfiguring daemon mon.a on smithi040 2023-07-14T11:04:07.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:07.867 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:04:08.263 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:08.263 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: Reconfiguring mon.b (monmap changed)... 2023-07-14T11:04:08.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:08.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:08.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:08.264 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:07 smithi130 ceph-mon[129405]: Reconfiguring daemon mon.b on smithi045 2023-07-14T11:04:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: Reconfiguring mon.b (monmap changed)... 2023-07-14T11:04:08.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:08.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:08.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:08.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:07 smithi040 ceph-mon[97954]: Reconfiguring daemon mon.b on smithi045 2023-07-14T11:04:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: Reconfiguring mon.b (monmap changed)... 2023-07-14T11:04:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:08.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:08.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:08.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:07 smithi045 ceph-mon[128791]: Reconfiguring daemon mon.b on smithi045 2023-07-14T11:04:09.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: Reconfiguring mon.c (monmap changed)... 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: Reconfiguring daemon mon.c on smithi130 2023-07-14T11:04:09.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:09 smithi130 ceph-mon[129405]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:09.411 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 -- 172.21.15.130:0/1580570222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 msgr2=0x7fbba0100030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:09.411 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 --2- 172.21.15.130:0/1580570222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 0x7fbba0100030 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fbb90009910 tx=0x7fbb90009c20 comp rx=0 tx=0).stop 2023-07-14T11:04:09.411 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 -- 172.21.15.130:0/1580570222 shutdown_connections 2023-07-14T11:04:09.411 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 --2- 172.21.15.130:0/1580570222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 0x7fbba0100030 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 -- 172.21.15.130:0/1580570222 >> 172.21.15.130:0/1580570222 conn(0x7fbba00f9320 msgr2=0x7fbba00fb770 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.401+0000 7fbba6570700 1 -- 172.21.15.130:0/1580570222 shutdown_connections 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 -- 172.21.15.130:0/1580570222 wait complete. 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 Processor -- start 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 -- start start 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 0x7fbba00fe2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:09.412 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbba019a120 0x7fbba019c5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbba019cd50 con 0x7fbba00ffc20 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbba019cec0 con 0x7fbba019a120 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.402+0000 7fbba6570700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbba019d080 con 0x7fbba01977b0 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39312/0 (socket says 172.21.15.130:39312) 2023-07-14T11:04:09.413 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 -- 172.21.15.130:0/3247221550 learned_addr learned my addr 172.21.15.130:0/3247221550 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 -- 172.21.15.130:0/3247221550 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbba019a120 msgr2=0x7fbba019c5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbba019a120 0x7fbba019c5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 -- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 msgr2=0x7fbba00fe2f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 0x7fbba00fe2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 -- 172.21.15.130:0/3247221550 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb900095f0 con 0x7fbba01977b0 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9f7fe700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800b780 tx=0x7fbb8800bb40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb88004230 con 0x7fbba01977b0 2023-07-14T11:04:09.414 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbba019d360 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb88004390 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb88011b30 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.403+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbba019d930 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.404+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7fbb8801a400 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.404+0000 7fbb9d7fa700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 0x7fbb8c03b0e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.404+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbba018d1a0 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.404+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7fbb8804b040 con 0x7fbba01977b0 2023-07-14T11:04:09.415 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.405+0000 7fbb9ffff700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 0x7fbb8c03b0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:09.416 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.405+0000 7fbb9ffff700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 0x7fbb8c03b0e0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fbb90005e00 tx=0x7fbb90005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:09.416 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.409+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fbb8800f8d0 con 0x7fbba01977b0 2023-07-14T11:04:09.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: Reconfiguring mon.c (monmap changed)... 2023-07-14T11:04:09.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:09.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:09.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:09.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: Reconfiguring daemon mon.c on smithi130 2023-07-14T11:04:09.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:09 smithi040 ceph-mon[97954]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:09.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: Reconfiguring mon.c (monmap changed)... 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: Reconfiguring daemon mon.c on smithi130 2023-07-14T11:04:09.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:09 smithi045 ceph-mon[128791]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:09.612 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.609+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi040=x;smithi045=y;smithi130=z", "target": ["mon-mgr", ""]}) v1 -- 0x7fbba00609e0 con 0x7fbb8c038c30 2023-07-14T11:04:09.620 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.616+0000 7fbb9d7fa700 1 -- 172.21.15.130:0/3247221550 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fbba00609e0 con 0x7fbb8c038c30 2023-07-14T11:04:09.620 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mgr update... 2023-07-14T11:04:09.622 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 msgr2=0x7fbb8c03b0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 0x7fbb8c03b0e0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fbb90005e00 tx=0x7fbb90005fd0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 msgr2=0x7fbba0199b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800b780 tx=0x7fbb8800bb40 comp rx=0 tx=0).stop 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 shutdown_connections 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb8c038c30 0x7fbb8c03b0e0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbba019a120 0x7fbba019c5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbba00ffc20 0x7fbba00fe2f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.624 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 --2- 172.21.15.130:0/3247221550 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbba01977b0 0x7fbba0199b90 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:09.624 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 >> 172.21.15.130:0/3247221550 conn(0x7fbba00f9320 msgr2=0x7fbba00fa070 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:09.624 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 shutdown_connections 2023-07-14T11:04:09.624 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:04:09.619+0000 7fbba6570700 1 -- 172.21.15.130:0/3247221550 wait complete. 2023-07-14T11:04:10.021 DEBUG:teuthology.orchestra.run.smithi045:mgr.y> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.y.service 2023-07-14T11:04:10.024 DEBUG:teuthology.orchestra.run.smithi130:mgr.z> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.z.service 2023-07-14T11:04:10.027 INFO:tasks.cephadm:Deploying OSDs... 2023-07-14T11:04:10.027 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:04:10.027 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/scratch_devs of=/dev/stdout 2023-07-14T11:04:10.056 INFO:journalctl@ceph.mgr.z.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:04:10.059 INFO:journalctl@ceph.mgr.y.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:04:10.075 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-07-14T11:04:10.076 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_1 2023-07-14T11:04:10.164 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-07-14T11:04:10.165 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 5h/5d Inode: 450 Links: 1 2023-07-14T11:04:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:10.165 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-07-14 11:03:16.276727338 +0000 2023-07-14T11:04:10.166 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-07-14 10:55:03.154574379 +0000 2023-07-14T11:04:10.166 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-07-14 10:55:03.154574379 +0000 2023-07-14T11:04:10.166 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: 2023-07-14 10:55:03.122574790 +0000 2023-07-14T11:04:10.166 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-07-14T11:04:10.235 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-07-14T11:04:10.235 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-07-14T11:04:10.236 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000298137 s, 1.7 MB/s 2023-07-14T11:04:10.238 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-07-14T11:04:10.284 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_2 2023-07-14T11:04:10.368 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-07-14T11:04:10.368 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:10.368 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 5h/5d Inode: 456 Links: 1 2023-07-14T11:04:10.368 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:10.368 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:10.369 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-07-14 11:03:16.276727338 +0000 2023-07-14T11:04:10.369 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-07-14 10:55:04.020563253 +0000 2023-07-14T11:04:10.369 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-07-14 10:55:04.020563253 +0000 2023-07-14T11:04:10.369 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: 2023-07-14 10:55:03.990563638 +0000 2023-07-14T11:04:10.369 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-07-14T11:04:10.474 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-07-14T11:04:10.474 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-07-14T11:04:10.474 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000288696 s, 1.8 MB/s 2023-07-14T11:04:10.476 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-07-14T11:04:10.522 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_3 2023-07-14T11:04:10.606 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-07-14T11:04:10.606 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 5h/5d Inode: 462 Links: 1 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-07-14 11:03:16.276727338 +0000 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-07-14 10:55:04.892552050 +0000 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-07-14 10:55:04.892552050 +0000 2023-07-14T11:04:10.607 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: 2023-07-14 10:55:04.866552385 +0000 2023-07-14T11:04:10.608 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-07-14T11:04:10.714 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-07-14T11:04:10.714 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-07-14T11:04:10.714 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000298093 s, 1.7 MB/s 2023-07-14T11:04:10.717 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-07-14T11:04:10.762 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_4 2023-07-14T11:04:10.847 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-07-14T11:04:10.847 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:10.847 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 5h/5d Inode: 468 Links: 1 2023-07-14T11:04:10.847 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:10.847 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:10.848 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-07-14 11:03:16.276727338 +0000 2023-07-14T11:04:10.848 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-07-14 10:55:05.765540835 +0000 2023-07-14T11:04:10.848 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-07-14 10:55:05.765540835 +0000 2023-07-14T11:04:10.848 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: 2023-07-14 10:55:05.736541207 +0000 2023-07-14T11:04:10.848 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-07-14T11:04:10.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi040=x;smithi045=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:10.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: Saving service mgr spec with placement smithi040=x;smithi045=y;smithi130=z;count:3 2023-07-14T11:04:10.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:10.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:10.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:10.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:10.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:10.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:10 smithi130 ceph-mon[129405]: Deploying daemon mgr.z on smithi130 2023-07-14T11:04:10.947 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi040=x;smithi045=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:10.948 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: Saving service mgr spec with placement smithi040=x;smithi045=y;smithi130=z;count:3 2023-07-14T11:04:10.948 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.948 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.949 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.949 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:10.949 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:10.950 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:10.950 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:10.950 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:10.950 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:10.951 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:10.951 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:10.951 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:10 smithi040 ceph-mon[97954]: Deploying daemon mgr.z on smithi130 2023-07-14T11:04:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-07-14T11:04:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-07-14T11:04:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000309683 s, 1.7 MB/s 2023-07-14T11:04:10.956 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-07-14T11:04:11.001 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2023-07-14T11:04:11.002 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/scratch_devs of=/dev/stdout 2023-07-14T11:04:11.024 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-07-14T11:04:11.024 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_1 2023-07-14T11:04:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "3;smithi040=x;smithi045=y;smithi130=z", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: Saving service mgr spec with placement smithi040=x;smithi045=y;smithi130=z;count:3 2023-07-14T11:04:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.z", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:11.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:11.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:11.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:10 smithi045 ceph-mon[128791]: Deploying daemon mgr.z on smithi130 2023-07-14T11:04:11.087 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-07-14T11:04:11.087 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 6h/6d Inode: 378246 Links: 1 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2023-07-14 11:03:56.533421887 +0000 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2023-07-14 10:51:47.618671080 +0000 2023-07-14T11:04:11.088 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2023-07-14 10:51:47.618671080 +0000 2023-07-14T11:04:11.089 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: - 2023-07-14T11:04:11.089 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-07-14T11:04:11.158 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2023-07-14T11:04:11.159 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2023-07-14T11:04:11.159 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000124237 s, 4.1 MB/s 2023-07-14T11:04:11.160 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-07-14T11:04:11.222 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_2 2023-07-14T11:04:11.282 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-07-14T11:04:11.282 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 6h/6d Inode: 376174 Links: 1 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2023-07-14 11:03:56.533421887 +0000 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2023-07-14 10:51:48.133662756 +0000 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2023-07-14 10:51:48.133662756 +0000 2023-07-14T11:04:11.283 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: - 2023-07-14T11:04:11.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-07-14T11:04:11.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 systemd[1]: Starting Ceph mgr.z for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:04:11.351 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2023-07-14T11:04:11.352 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2023-07-14T11:04:11.352 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000115322 s, 4.4 MB/s 2023-07-14T11:04:11.353 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-07-14T11:04:11.412 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_3 2023-07-14T11:04:11.471 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 6h/6d Inode: 378440 Links: 1 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2023-07-14 11:03:56.533421887 +0000 2023-07-14T11:04:11.472 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2023-07-14 10:51:48.685653834 +0000 2023-07-14T11:04:11.473 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2023-07-14 10:51:48.685653834 +0000 2023-07-14T11:04:11.473 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: - 2023-07-14T11:04:11.473 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-07-14T11:04:11.540 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2023-07-14T11:04:11.540 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2023-07-14T11:04:11.540 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 9.9572e-05 s, 5.1 MB/s 2023-07-14T11:04:11.541 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-07-14T11:04:11.600 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_4 2023-07-14T11:04:11.658 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 6h/6d Inode: 378549 Links: 1 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2023-07-14 11:03:56.533421887 +0000 2023-07-14T11:04:11.659 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2023-07-14 10:51:49.216645252 +0000 2023-07-14T11:04:11.660 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2023-07-14 10:51:49.216645252 +0000 2023-07-14T11:04:11.660 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: - 2023-07-14T11:04:11.660 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-07-14T11:04:11.682 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 podman[131231]: 2023-07-14T11:04:11.682 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 bash[131231]: c7b2f42792aead68f94a19c5edceecac8b4bfbbe99386c8be55e29b1816057de 2023-07-14T11:04:11.682 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-mgr[131271]: -- 172.21.15.130:0/1081523762 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558310d471e0 con 0x558310d1c800 2023-07-14T11:04:11.726 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2023-07-14T11:04:11.726 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2023-07-14T11:04:11.726 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000115068 s, 4.4 MB/s 2023-07-14T11:04:11.727 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-07-14T11:04:11.786 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2023-07-14T11:04:11.786 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2023-07-14T11:04:11.803 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-07-14T11:04:11.804 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2023-07-14T11:04:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:11 smithi045 ceph-mon[128791]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:11.861 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-07-14T11:04:11.861 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:11.861 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 6h/6d Inode: 378232 Links: 1 2023-07-14T11:04:11.862 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:11.862 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:11.862 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-07-14 11:04:11.043960367 +0000 2023-07-14T11:04:11.862 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-07-14 10:51:20.828587325 +0000 2023-07-14T11:04:11.863 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-07-14 10:51:20.828587325 +0000 2023-07-14T11:04:11.863 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-07-14T11:04:11.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-07-14T11:04:11.931 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-07-14T11:04:11.931 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-07-14T11:04:11.932 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.00025182 s, 2.0 MB/s 2023-07-14T11:04:11.932 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-07-14T11:04:11.989 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2023-07-14T11:04:12.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-mon[129405]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:12.002 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 systemd[1]: Started Ceph mgr.z for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:04:12.002 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:11.716+0000 7f0705c1b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:04:12.002 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:11.807+0000 7f0705c1b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:04:12.002 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:11.898+0000 7f0705c1b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:04:12.018 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-07-14T11:04:12.019 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:12.019 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 6h/6d Inode: 365366 Links: 1 2023-07-14T11:04:12.019 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:12.019 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:12.019 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-07-14 11:04:11.043960367 +0000 2023-07-14T11:04:12.020 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-07-14 10:51:21.404577109 +0000 2023-07-14T11:04:12.020 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-07-14 10:51:21.404577109 +0000 2023-07-14T11:04:12.020 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-07-14T11:04:12.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-07-14T11:04:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:11 smithi040 ceph-mon[97954]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:12.086 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-07-14T11:04:12.087 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-07-14T11:04:12.087 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000231807 s, 2.2 MB/s 2023-07-14T11:04:12.088 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-07-14T11:04:12.150 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2023-07-14T11:04:12.208 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-07-14T11:04:12.208 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 6h/6d Inode: 375468 Links: 1 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-07-14 11:04:11.043960367 +0000 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-07-14 10:51:21.956567318 +0000 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-07-14 10:51:21.956567318 +0000 2023-07-14T11:04:12.209 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-07-14T11:04:12.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-07-14T11:04:12.271 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:11.998+0000 7f0705c1b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:04:12.271 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:12.180+0000 7f0705c1b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:04:12.273 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-07-14T11:04:12.274 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-07-14T11:04:12.274 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000219422 s, 2.3 MB/s 2023-07-14T11:04:12.275 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-07-14T11:04:12.333 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2023-07-14T11:04:12.390 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-07-14T11:04:12.390 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-07-14T11:04:12.390 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 6h/6d Inode: 378367 Links: 1 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2023-07-14 11:04:11.043960367 +0000 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2023-07-14 10:51:22.486557918 +0000 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2023-07-14 10:51:22.486557918 +0000 2023-07-14T11:04:12.391 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: - 2023-07-14T11:04:12.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-07-14T11:04:12.456 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2023-07-14T11:04:12.456 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2023-07-14T11:04:12.456 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000175144 s, 2.9 MB/s 2023-07-14T11:04:12.457 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-07-14T11:04:12.514 INFO:tasks.cephadm:Deploying osd.0 on smithi040 with /dev/vg_nvme/lv_4... 2023-07-14T11:04:12.515 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-07-14T11:04:12.542 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:12.504+0000 7f0705c1b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:04:12.953 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:12.954 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:12.954 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:12.954 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:12.954 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:12.954 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:12.955 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:12.955 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:12 smithi045 ceph-mon[128791]: Deploying daemon mgr.y on smithi045 2023-07-14T11:04:13.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:12.594+0000 7f0705c1b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:04:13.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:12.722+0000 7f0705c1b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:04:13.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:12.817+0000 7f0705c1b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:04:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:12 smithi130 ceph-mon[129405]: Deploying daemon mgr.y on smithi045 2023-07-14T11:04:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-07-14T11:04:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:12 smithi040 ceph-mon[97954]: Deploying daemon mgr.y on smithi045 2023-07-14T11:04:13.252 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:04:13.545 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:13.234+0000 7f0705c1b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:04:13.565 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:13 smithi045 systemd[1]: Starting Ceph mgr.y for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:04:13.973 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:13 smithi045 ceph-mon[128791]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:13.973 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:13 smithi045 podman[130226]: 2023-07-14T11:04:13.974 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:13 smithi045 ceph-mgr[130266]: -- 172.21.15.45:0/2506396569 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563c889391e0 con 0x563c8890f000 2023-07-14T11:04:13.974 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:13 smithi045 bash[130226]: 07fcc2041cd9bd5ab2a17f75291559b2485efdea7ebcda810eb4547ebb342478 2023-07-14T11:04:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:13 smithi130 ceph-mon[129405]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:14.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:13.805+0000 7f0705c1b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:04:14.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:13.975+0000 7f0705c1b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:04:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:13 smithi040 ceph-mon[97954]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:14.293 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:13 smithi045 systemd[1]: Started Ceph mgr.y for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:04:14.294 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:14.085+0000 7ffabc8e9200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:04:14.294 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:14.180+0000 7ffabc8e9200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:04:14.294 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:14.291+0000 7ffabc8e9200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:04:14.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:14.406+0000 7ffabc8e9200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:04:15.025 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:14.617+0000 7ffabc8e9200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:04:15.034 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:14 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:14.767+0000 7f0705c1b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:04:15.034 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:14 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:14.857+0000 7f0705c1b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:04:15.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:15.037+0000 7f0705c1b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:04:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-mon[129405]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:15.294 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:15.023+0000 7ffabc8e9200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:04:15.294 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:15.136+0000 7ffabc8e9200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:15.295 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-mon[128791]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:15.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:15.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:15.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:15 smithi040 ceph-mon[97954]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:15.368 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-07-14T11:04:15.368 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-07-14T11:04:15.369 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10+0 records in 2023-07-14T11:04:15.369 INFO:teuthology.orchestra.run.smithi040.stderr:10+0 records out 2023-07-14T11:04:15.369 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0308066 s, 340 MB/s 2023-07-14T11:04:15.369 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping successful for: 2023-07-14T11:04:15.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:15.292+0000 7ffabc8e9200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:04:15.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:15.411+0000 7ffabc8e9200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:04:15.740 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:15.332+0000 7f0705c1b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:04:15.741 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:15.577+0000 7f0705c1b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:04:15.741 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:15.738+0000 7f0705c1b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:04:15.841 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi040:vg_nvme/lv_4 2023-07-14T11:04:16.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:15.998+0000 7f0705c1b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:04:16.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:15.938+0000 7ffabc8e9200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:04:16.598 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.013 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:17.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:17.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:17.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.015 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:16.697+0000 7f0705c1b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:04:17.015 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:16.825+0000 7f0705c1b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:04:17.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.016 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:17.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:17.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.017 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.064 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:16.633+0000 7ffabc8e9200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:04:17.064 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:16.824+0000 7ffabc8e9200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:04:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-07-14T11:04:17.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-07-14T11:04:17.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:17.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:17.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:17.012+0000 7f0705c1b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:04:17.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:17.126+0000 7f0705c1b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:04:17.793 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:17.337+0000 7f0705c1b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:04:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:17 smithi045 ceph-mon[128791]: Reconfiguring mgr.x (unknown last config time)... 2023-07-14T11:04:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:17 smithi045 ceph-mon[128791]: Reconfiguring daemon mgr.x on smithi040 2023-07-14T11:04:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:17 smithi045 ceph-mon[128791]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:18.065 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:17.785+0000 7ffabc8e9200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:04:18.065 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:17.897+0000 7ffabc8e9200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:04:18.188 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-mon[129405]: Reconfiguring mgr.x (unknown last config time)... 2023-07-14T11:04:18.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-mon[129405]: Reconfiguring daemon mgr.x on smithi040 2023-07-14T11:04:18.189 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:17 smithi130 ceph-mon[129405]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:18.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:17 smithi040 ceph-mon[97954]: Reconfiguring mgr.x (unknown last config time)... 2023-07-14T11:04:18.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:17 smithi040 ceph-mon[97954]: Reconfiguring daemon mgr.x on smithi040 2023-07-14T11:04:18.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:17 smithi040 ceph-mon[97954]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:18.462 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:18.114+0000 7ffabc8e9200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:04:18.543 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:04:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:04:18.185+0000 7f0705c1b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:04:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 -- 172.21.15.40:0/2680436689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 msgr2=0x7f6fd80729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2680436689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd80729a0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f6fc8009990 tx=0x7f6fc8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 -- 172.21.15.40:0/2680436689 shutdown_connections 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2680436689 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 0x7f6fd8109ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2680436689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd80729a0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2680436689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd80733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.600+0000 7f6fdca05700 1 -- 172.21.15.40:0/2680436689 >> 172.21.15.40:0/2680436689 conn(0x7f6fd80fb1a0 msgr2=0x7f6fd80fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.601+0000 7f6fdca05700 1 -- 172.21.15.40:0/2680436689 shutdown_connections 2023-07-14T11:04:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.601+0000 7f6fdca05700 1 -- 172.21.15.40:0/2680436689 wait complete. 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 Processor -- start 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 -- start start 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd8198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 0x7f6fd819f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819f660 con 0x7f6fd8074540 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819f7a0 con 0x7f6fd8107a60 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fdca05700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fd819f930 con 0x7f6fd8072f70 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fd659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fd5d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd8198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fd659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60186/0 (socket says 172.21.15.40:60186) 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fd659c700 1 -- 172.21.15.40:0/2035748376 learned_addr learned my addr 172.21.15.40:0/2035748376 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:04:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.602+0000 7f6fd6d9d700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 0x7f6fd819f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 -- 172.21.15.40:0/2035748376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 msgr2=0x7f6fd819f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 0x7f6fd819f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 -- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 msgr2=0x7f6fd8198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd8198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 -- 172.21.15.40:0/2035748376 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fc80095f0 con 0x7f6fd8072f70 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fd659c700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6fc8004770 tx=0x7f6fc8009580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fc801d070 con 0x7f6fd8072f70 2023-07-14T11:04:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6fc8005030 con 0x7f6fd8072f70 2023-07-14T11:04:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fd819fbb0 con 0x7f6fd8072f70 2023-07-14T11:04:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.603+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6fc8004b50 con 0x7f6fd8072f70 2023-07-14T11:04:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.604+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fd819ff60 con 0x7f6fd8072f70 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.605+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 13) v1 ==== 45539+0+0 (secure 0 0 0) 0x7f6fc8004cf0 con 0x7f6fd8072f70 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.605+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fd8106110 con 0x7f6fd8072f70 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.605+0000 7f6fc77fe700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 0x7f6fc003ae90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.605+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1155+0+0 (secure 0 0 0) 0x7f6fc804c590 con 0x7f6fd8072f70 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.605+0000 7f6fd5d9b700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 0x7f6fc003ae90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.606+0000 7f6fd5d9b700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 0x7f6fc003ae90 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6fcc009c60 tx=0x7f6fcc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:18.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.610+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6fc8029950 con 0x7f6fd8072f70 2023-07-14T11:04:18.713 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:18.460+0000 7ffabc8e9200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:04:18.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:18.826+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f6fd8060950 con 0x7f6fc00389e0 2023-07-14T11:04:19.036 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:18.710+0000 7ffabc8e9200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:04:19.037 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:18.891+0000 7ffabc8e9200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:04:19.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:19.054+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mgrmap(e 14) v1 ==== 90599+0+0 (secure 0 0 0) 0x7f6fc8021400 con 0x7f6fd8072f70 2023-07-14T11:04:19.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:19.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:19.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:19.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: Standby manager daemon z started 2023-07-14T11:04:19.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:04:19.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:04:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:19.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:19.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:19.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:19.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:19.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:19.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: Standby manager daemon z started 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:19.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:19.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:19.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:19.216+0000 7ffabc8e9200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:04:19.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: Standby manager daemon z started 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:04:19.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/1771083988' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='client.24121 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:19.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:20.253 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:20 smithi040 ceph-mon[97954]: mgrmap e14: x(active, since 72s), standbys: z 2023-07-14T11:04:20.253 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:20 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:04:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-mon[128791]: mgrmap e14: x(active, since 72s), standbys: z 2023-07-14T11:04:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:04:20.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:20.069+0000 7ffabc8e9200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:04:20.315 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:20.226+0000 7ffabc8e9200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:04:20.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:20 smithi130 ceph-mon[129405]: mgrmap e14: x(active, since 72s), standbys: z 2023-07-14T11:04:20.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:20 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:04:20.814 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:20.466+0000 7ffabc8e9200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:04:20.814 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:20.602+0000 7ffabc8e9200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:04:21.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:21 smithi040 ceph-mon[97954]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:21.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:21 smithi045 ceph-mon[128791]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:21.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:20.872+0000 7ffabc8e9200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:04:21.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:21 smithi130 ceph-mon[129405]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:22.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:04:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:04:21.905+0000 7ffabc8e9200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]: dispatch 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]': finished 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: osdmap e5: 1 total, 0 up, 1 in 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: Standby manager daemon y started 2023-07-14T11:04:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:04:22.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:22.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:04:22.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:22 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:22.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]: dispatch 2023-07-14T11:04:22.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]': finished 2023-07-14T11:04:22.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: osdmap e5: 1 total, 0 up, 1 in 2023-07-14T11:04:22.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:22.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: Standby manager daemon y started 2023-07-14T11:04:22.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:04:22.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:22.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:04:22.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:22 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]: dispatch 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1043181718' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ec960268-07b9-478c-9217-22b4c81f02b2"}]': finished 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: osdmap e5: 1 total, 0 up, 1 in 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: Standby manager daemon y started 2023-07-14T11:04:22.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:04:22.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:04:22.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:04:22.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:22 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/3198239505' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:04:22.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:22.924+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mon.1 v2:172.21.15.130:3300/0 8 ==== mgrmap(e 15) v1 ==== 135659+0+0 (secure 0 0 0) 0x7f6fc80051a0 con 0x7f6fd8072f70 2023-07-14T11:04:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:23 smithi130 ceph-mon[129405]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:23 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2781891357' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:23 smithi130 ceph-mon[129405]: mgrmap e15: x(active, since 76s), standbys: z, y 2023-07-14T11:04:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:04:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:23 smithi040 ceph-mon[97954]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:23 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2781891357' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:23 smithi040 ceph-mon[97954]: mgrmap e15: x(active, since 76s), standbys: z, y 2023-07-14T11:04:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:04:23.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:23 smithi045 ceph-mon[128791]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:23 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2781891357' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:23 smithi045 ceph-mon[128791]: mgrmap e15: x(active, since 76s), standbys: z, y 2023-07-14T11:04:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:04:25.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:25 smithi130 ceph-mon[129405]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:25.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:25 smithi040 ceph-mon[97954]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:25 smithi045 ceph-mon[128791]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:27.786 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:27 smithi040 ceph-mon[97954]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:27 smithi130 ceph-mon[129405]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:27 smithi045 ceph-mon[128791]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:29 smithi130 ceph-mon[129405]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:29 smithi040 ceph-mon[97954]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:29 smithi045 ceph-mon[128791]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:30.953 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-07-14T11:04:30.953 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:30.953 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:30 smithi040 ceph-mon[97954]: Deploying daemon osd.0 on smithi040 2023-07-14T11:04:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-07-14T11:04:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:30 smithi130 ceph-mon[129405]: Deploying daemon osd.0 on smithi040 2023-07-14T11:04:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-07-14T11:04:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:30 smithi045 ceph-mon[128791]: Deploying daemon osd.0 on smithi040 2023-07-14T11:04:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:31 smithi130 ceph-mon[129405]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:31 smithi040 ceph-mon[97954]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:31 smithi045 ceph-mon[128791]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:33 smithi040 ceph-mon[97954]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:33 smithi130 ceph-mon[129405]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:33 smithi045 ceph-mon[128791]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:35.318 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:35 smithi040 ceph-mon[97954]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:35.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:35 smithi130 ceph-mon[129405]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:35.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:35 smithi045 ceph-mon[128791]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:37.722 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:37.722 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.722 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.723 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:37 smithi040 ceph-mon[97954]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:37 smithi130 ceph-mon[129405]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:37 smithi045 ceph-mon[128791]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:40.291 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:40.292 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.292 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.292 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:40.292 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:40.292 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:40.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:40.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.289+0000 7f6fc77fe700 1 -- 172.21.15.40:0/2035748376 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f6fd8060950 con 0x7f6fc00389e0 2023-07-14T11:04:40.294 INFO:teuthology.orchestra.run.smithi040.stdout:Created osd(s) 0 on host 'smithi040' 2023-07-14T11:04:40.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 msgr2=0x7f6fc003ae90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:40.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 0x7f6fc003ae90 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6fcc009c60 tx=0x7f6fcc009260 comp rx=0 tx=0).stop 2023-07-14T11:04:40.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 msgr2=0x7f6fd8197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f6fc8004770 tx=0x7f6fc8009580 comp rx=0 tx=0).stop 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 shutdown_connections 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6fc00389e0 0x7f6fc003ae90 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6fd8107a60 0x7f6fd819f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6fd8072f70 0x7f6fd8197ad0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 --2- 172.21.15.40:0/2035748376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fd8074540 0x7f6fd8198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 >> 172.21.15.40:0/2035748376 conn(0x7f6fd80fb1a0 msgr2=0x7f6fd80fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 shutdown_connections 2023-07-14T11:04:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:04:40.292+0000 7f6fdca05700 1 -- 172.21.15.40:0/2035748376 wait complete. 2023-07-14T11:04:40.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:40.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:40.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:40.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:40.814 DEBUG:teuthology.orchestra.run.smithi040:osd.0> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.0.service 2023-07-14T11:04:40.818 INFO:tasks.cephadm:Deploying osd.1 on smithi045 with /dev/vg_nvme/lv_4... 2023-07-14T11:04:40.819 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-07-14T11:04:40.893 INFO:journalctl@ceph.osd.0.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:04:41.389 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:04:41.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:41 smithi130 ceph-mon[129405]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-07-14T11:04:41.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:41 smithi130 ceph-mon[129405]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:41.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:41 smithi040 ceph-mon[97954]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-07-14T11:04:41.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:41 smithi040 ceph-mon[97954]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:41.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:41.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:04:41.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:41 smithi045 ceph-mon[128791]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-07-14T11:04:41.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:41 smithi045 ceph-mon[128791]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:42.400 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-mon[97954]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-07-14T11:04:42.400 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-mon[97954]: osdmap e6: 1 total, 0 up, 1 in 2023-07-14T11:04:42.401 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:42.401 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-mon[97954]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:04:42.401 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0[110217]: 2023-07-14T11:04:42.271+0000 7f428e7d0700 -1 osd.0 0 waiting for initial osdmap 2023-07-14T11:04:42.401 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:04:42 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0[110217]: 2023-07-14T11:04:42.276+0000 7f42865bd700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:04:42.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:42 smithi130 ceph-mon[129405]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-07-14T11:04:42.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:42 smithi130 ceph-mon[129405]: osdmap e6: 1 total, 0 up, 1 in 2023-07-14T11:04:42.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:42.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:42 smithi130 ceph-mon[129405]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:04:42.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:42 smithi045 ceph-mon[128791]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-07-14T11:04:42.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:42 smithi045 ceph-mon[128791]: osdmap e6: 1 total, 0 up, 1 in 2023-07-14T11:04:42.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:42.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:42 smithi045 ceph-mon[128791]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:04:42.896 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-07-14T11:04:42.896 INFO:teuthology.orchestra.run.smithi045.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-07-14T11:04:42.896 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10+0 records in 2023-07-14T11:04:42.897 INFO:teuthology.orchestra.run.smithi045.stderr:10+0 records out 2023-07-14T11:04:42.897 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0131518 s, 797 MB/s 2023-07-14T11:04:42.897 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping successful for: 2023-07-14T11:04:43.331 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi045:vg_nvme/lv_4 2023-07-14T11:04:43.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:43 smithi130 ceph-mon[129405]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:04:43.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:43 smithi130 ceph-mon[129405]: osdmap e7: 1 total, 0 up, 1 in 2023-07-14T11:04:43.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:43 smithi130 ceph-mon[129405]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:43 smithi040 ceph-mon[97954]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:04:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:43 smithi040 ceph-mon[97954]: osdmap e7: 1 total, 0 up, 1 in 2023-07-14T11:04:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:43 smithi040 ceph-mon[97954]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:43.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:43 smithi045 ceph-mon[128791]: from='osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:04:43.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:43 smithi045 ceph-mon[128791]: osdmap e7: 1 total, 0 up, 1 in 2023-07-14T11:04:43.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:43.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:43 smithi045 ceph-mon[128791]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-07-14T11:04:43.888 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:04:44.526 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:04:44.527 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:04:44.527 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.527 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] boot 2023-07-14T11:04:44.527 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: osdmap e8: 1 total, 1 up, 1 in 2023-07-14T11:04:44.527 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi040 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 17272M 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.528 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:44.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:44.529 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:04:44.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:04:44.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] boot 2023-07-14T11:04:44.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: osdmap e8: 1 total, 1 up, 1 in 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi040 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 17272M 2023-07-14T11:04:44.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:44.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:44.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:04:44.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:04:44.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: osd.0 [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] boot 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: osdmap e8: 1 total, 1 up, 1 in 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi040 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:04:44.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 17272M 2023-07-14T11:04:44.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:44.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:44.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:04:44.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:04:45.305 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 -- 172.21.15.45:0/3547424397 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 msgr2=0x7fc9f4107cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:45.305 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 --2- 172.21.15.45:0/3547424397 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f4107cb0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec00b0a0 tx=0x7fc9ec00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.305 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 -- 172.21.15.45:0/3547424397 shutdown_connections 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 --2- 172.21.15.45:0/3547424397 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f4107cb0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 --2- 172.21.15.45:0/3547424397 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4100670 0x7fc9f4102a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 --2- 172.21.15.45:0/3547424397 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4102f90 0x7fc9f4105370 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 -- 172.21.15.45:0/3547424397 >> 172.21.15.45:0/3547424397 conn(0x7fc9f40fa0b0 msgr2=0x7fc9f40fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 -- 172.21.15.45:0/3547424397 shutdown_connections 2023-07-14T11:04:45.306 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.302+0000 7fc9f8946700 1 -- 172.21.15.45:0/3547424397 wait complete. 2023-07-14T11:04:45.308 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 Processor -- start 2023-07-14T11:04:45.308 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 -- start start 2023-07-14T11:04:45.308 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4100670 0x7fc9f4195aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:45.308 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 0x7fc9f4195fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:45.308 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9f419d5e0 con 0x7fc9f4102f90 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9f419d720 con 0x7fc9f41058b0 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f8946700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc9f419d8b0 con 0x7fc9f4100670 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.45:53942/0 (socket says 172.21.15.45:53942) 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 -- 172.21.15.45:0/934767806 learned_addr learned my addr 172.21.15.45:0/934767806 (peer_addr_for_me v2:172.21.15.45:0/0) 2023-07-14T11:04:45.309 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f259c700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4100670 0x7fc9f4195aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 -- 172.21.15.45:0/934767806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4100670 msgr2=0x7fc9f4195aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4100670 0x7fc9f4195aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 -- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 msgr2=0x7fc9f4195fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f1d9b700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 0x7fc9f4195fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 0x7fc9f4195fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f2d9d700 1 -- 172.21.15.45:0/934767806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc9ec009d00 con 0x7fc9f41058b0 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.303+0000 7fc9f1d9b700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 0x7fc9f4195fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:04:45.310 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9f2d9d700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec009fd0 tx=0x7fc9ec004c80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec00e040 con 0x7fc9f41058b0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc9ec005710 con 0x7fc9f41058b0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc9ec01db60 con 0x7fc9f41058b0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc9f419db30 con 0x7fc9f41058b0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.304+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc9f419dfc0 con 0x7fc9f41058b0 2023-07-14T11:04:45.311 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.305+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc9f40fbc20 con 0x7fc9f41058b0 2023-07-14T11:04:45.312 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.306+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 15) v1 ==== 135659+0+0 (secure 0 0 0) 0x7fc9ec005880 con 0x7fc9f41058b0 2023-07-14T11:04:45.312 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.306+0000 7fc9eb7fe700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 0x7fc9e00a35f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:04:45.313 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.307+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(9..9 src has 1..9) v4 ==== 1767+0+0 (secure 0 0 0) 0x7fc9ec0ccc70 con 0x7fc9f41058b0 2023-07-14T11:04:45.313 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.307+0000 7fc9f259c700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 0x7fc9e00a35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:04:45.313 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.307+0000 7fc9f259c700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 0x7fc9e00a35f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fc9e4009a10 tx=0x7fc9e4009190 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:04:45.313 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.310+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc9ec097ec0 con 0x7fc9f41058b0 2023-07-14T11:04:45.516 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:04:45.513+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7fc9f4060950 con 0x7fc9e00a1140 2023-07-14T11:04:45.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:45 smithi130 ceph-mon[129405]: osdmap e9: 1 total, 1 up, 1 in 2023-07-14T11:04:45.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:45 smithi130 ceph-mon[129405]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:45.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:45 smithi040 ceph-mon[97954]: osdmap e9: 1 total, 1 up, 1 in 2023-07-14T11:04:45.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:45 smithi040 ceph-mon[97954]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:45.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:45 smithi045 ceph-mon[128791]: osdmap e9: 1 total, 1 up, 1 in 2023-07-14T11:04:45.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:45 smithi045 ceph-mon[128791]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:46.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:46 smithi130 ceph-mon[129405]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:46.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:46.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:46.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:46 smithi040 ceph-mon[97954]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:46.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:46 smithi045 ceph-mon[128791]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:04:46.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:04:46.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:04:46.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:47.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:47 smithi045 ceph-mon[128791]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:47 smithi130 ceph-mon[129405]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:47.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:47 smithi040 ceph-mon[97954]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:48.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:48 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/2421418848' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:48 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:48 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]': finished 2023-07-14T11:04:48.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:48 smithi045 ceph-mon[128791]: osdmap e10: 2 total, 1 up, 2 in 2023-07-14T11:04:48.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:48 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:04:48.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/2421418848' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:48 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:48 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]': finished 2023-07-14T11:04:48.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:48 smithi130 ceph-mon[129405]: osdmap e10: 2 total, 1 up, 2 in 2023-07-14T11:04:48.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:48 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:04:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:48 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/2421418848' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:48 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]: dispatch 2023-07-14T11:04:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:48 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d1f5898-462a-4229-8a58-4c31ab9a6b13"}]': finished 2023-07-14T11:04:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:48 smithi040 ceph-mon[97954]: osdmap e10: 2 total, 1 up, 2 in 2023-07-14T11:04:48.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:48 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:04:49.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:49 smithi045 ceph-mon[128791]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:49.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/1714279839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:49 smithi130 ceph-mon[129405]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/1714279839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:49 smithi040 ceph-mon[97954]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/1714279839' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:04:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:51 smithi130 ceph-mon[129405]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:51 smithi040 ceph-mon[97954]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:51 smithi045 ceph-mon[128791]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:53 smithi130 ceph-mon[129405]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:53 smithi040 ceph-mon[97954]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:53 smithi045 ceph-mon[128791]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:55 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-07-14T11:04:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:55 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:55 smithi045 ceph-mon[128791]: Deploying daemon osd.1 on smithi045 2023-07-14T11:04:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:55 smithi045 ceph-mon[128791]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:55.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:55 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-07-14T11:04:55.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:55 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:55.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:55 smithi130 ceph-mon[129405]: Deploying daemon osd.1 on smithi045 2023-07-14T11:04:55.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:55 smithi130 ceph-mon[129405]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:55 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-07-14T11:04:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:55 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:04:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:55 smithi040 ceph-mon[97954]: Deploying daemon osd.1 on smithi045 2023-07-14T11:04:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:55 smithi040 ceph-mon[97954]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:57.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:57 smithi045 ceph-mon[128791]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:57 smithi130 ceph-mon[129405]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:04:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:57 smithi040 ceph-mon[97954]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:59 smithi045 ceph-mon[128791]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:04:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:59 smithi130 ceph-mon[129405]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:04:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:59 smithi040 ceph-mon[97954]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:04:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:01.733 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 1 on host 'smithi045' 2023-07-14T11:05:01.734 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.731+0000 7fc9eb7fe700 1 -- 172.21.15.45:0/934767806 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fc9f4060950 con 0x7fc9e00a1140 2023-07-14T11:05:01.735 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.733+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 msgr2=0x7fc9e00a35f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:01.735 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.733+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 0x7fc9e00a35f0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fc9e4009a10 tx=0x7fc9e4009190 comp rx=0 tx=0).stop 2023-07-14T11:05:01.736 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.733+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 msgr2=0x7fc9f419d010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:01.736 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.733+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fc9ec009fd0 tx=0x7fc9ec004c80 comp rx=0 tx=0).stop 2023-07-14T11:05:01.736 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 shutdown_connections 2023-07-14T11:05:01.736 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc9e00a1140 0x7fc9e00a35f0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:01.736 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc9f41058b0 0x7fc9f419d010 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:01.737 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc9f4100670 0x7fc9f4195aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:01.737 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 --2- 172.21.15.45:0/934767806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9f4102f90 0x7fc9f4195fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:01.737 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 >> 172.21.15.45:0/934767806 conn(0x7fc9f40fa0b0 msgr2=0x7fc9f41012f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:01.737 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 shutdown_connections 2023-07-14T11:05:01.737 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:01.734+0000 7fc9f8946700 1 -- 172.21.15.45:0/934767806 wait complete. 2023-07-14T11:05:02.225 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:02.225 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.226 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.282 DEBUG:teuthology.orchestra.run.smithi045:osd.1> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.1.service 2023-07-14T11:05:02.284 INFO:tasks.cephadm:Deploying osd.2 on smithi130 with /dev/vg_nvme/lv_4... 2023-07-14T11:05:02.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-07-14T11:05:02.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:02.318 INFO:journalctl@ceph.osd.1.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:05:02.846 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:05:02.944 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:05:02.923+0000 7fe19656f640 -1 osd.1 0 log_to_monitors true 2023-07-14T11:05:03.226 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:02 smithi130 ceph-mon[129405]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:03.227 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:02 smithi130 ceph-mon[129405]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:03.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:02 smithi045 ceph-mon[128791]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:03.275 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:02 smithi045 ceph-mon[128791]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:03.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:02 smithi040 ceph-mon[97954]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:03.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:02 smithi040 ceph-mon[97954]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-07-14T11:05:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-07-14T11:05:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: osdmap e11: 2 total, 1 up, 2 in 2023-07-14T11:05:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-mon[128791]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.066 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:05:03.953+0000 7fe18bbe7700 -1 osd.1 0 waiting for initial osdmap 2023-07-14T11:05:04.066 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:05:03.956+0000 7fe1841d5700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:05:04.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-07-14T11:05:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: osdmap e11: 2 total, 1 up, 2 in 2023-07-14T11:05:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:03 smithi130 ceph-mon[129405]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: pgmap v58: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-07-14T11:05:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: osdmap e11: 2 total, 1 up, 2 in 2023-07-14T11:05:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: from='osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:03 smithi040 ceph-mon[97954]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:05:04.333 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-07-14T11:05:04.334 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-07-14T11:05:04.334 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-07-14T11:05:04.334 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-07-14T11:05:04.334 INFO:teuthology.orchestra.run.smithi130.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0146295 s, 717 MB/s 2023-07-14T11:05:04.334 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-07-14T11:05:04.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2023-07-14T11:05:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:05:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: osdmap e12: 2 total, 1 up, 2 in 2023-07-14T11:05:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:05.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:05:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: osdmap e12: 2 total, 1 up, 2 in 2023-07-14T11:05:05.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:05:05.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: osdmap e12: 2 total, 1 up, 2 in 2023-07-14T11:05:05.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:05.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:05.393 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:05:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:05:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:05:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi045 2023-07-14T11:05:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 17173M 2023-07-14T11:05:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] boot 2023-07-14T11:05:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: osdmap e13: 2 total, 2 up, 2 in 2023-07-14T11:05:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:05:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:05:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi045 2023-07-14T11:05:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 17173M 2023-07-14T11:05:06.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] boot 2023-07-14T11:05:06.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: osdmap e13: 2 total, 2 up, 2 in 2023-07-14T11:05:06.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:05:06.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: pgmap v61: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi045 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 17173M 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: osd.1 [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] boot 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: osdmap e13: 2 total, 2 up, 2 in 2023-07-14T11:05:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:05:06.635 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 -- 172.21.15.130:0/1406160323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640fea80 msgr2=0x7fdb640fee90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:06.636 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 --2- 172.21.15.130:0/1406160323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640fea80 0x7fdb640fee90 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fdb58009990 tx=0x7fdb58009ca0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.636 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 -- 172.21.15.130:0/1406160323 shutdown_connections 2023-07-14T11:05:06.636 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 --2- 172.21.15.130:0/1406160323 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb64100610 0x7fdb64106ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.636 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 --2- 172.21.15.130:0/1406160323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640fea80 0x7fdb640fee90 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.636 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 --2- 172.21.15.130:0/1406160323 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb640ffc80 0x7fdb641000d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.637 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 -- 172.21.15.130:0/1406160323 >> 172.21.15.130:0/1406160323 conn(0x7fdb640fa0b0 msgr2=0x7fdb640fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:06.637 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 -- 172.21.15.130:0/1406160323 shutdown_connections 2023-07-14T11:05:06.637 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.633+0000 7fdb69966700 1 -- 172.21.15.130:0/1406160323 wait complete. 2023-07-14T11:05:06.638 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 Processor -- start 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 -- start start 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 0x7fdb64194860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb64100610 0x7fdb6419be20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb6419c3f0 con 0x7fdb64100610 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb6419c530 con 0x7fdb640fea80 2023-07-14T11:05:06.639 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb69966700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdb6419c6c0 con 0x7fdb640ffc80 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39342/0 (socket says 172.21.15.130:39342) 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 -- 172.21.15.130:0/2973524099 learned_addr learned my addr 172.21.15.130:0/2973524099 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 -- 172.21.15.130:0/2973524099 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 msgr2=0x7fdb64194860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb62ffd700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 0x7fdb64194860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb637fe700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb64100610 0x7fdb6419be20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:06.640 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 0x7fdb64194860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 -- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb64100610 msgr2=0x7fdb6419be20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb64100610 0x7fdb6419be20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 -- 172.21.15.130:0/2973524099 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdb580095f0 con 0x7fdb640ffc80 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb62ffd700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 0x7fdb64194860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.634+0000 7fdb627fc700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fdb5400b560 tx=0x7fdb5400b920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.635+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb54011830 con 0x7fdb640ffc80 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.635+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdb54011e50 con 0x7fdb640ffc80 2023-07-14T11:05:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.635+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdb6419c9a0 con 0x7fdb640ffc80 2023-07-14T11:05:06.642 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.635+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdb5400f2e0 con 0x7fdb640ffc80 2023-07-14T11:05:06.642 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.635+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdb6419cf20 con 0x7fdb640ffc80 2023-07-14T11:05:06.642 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.636+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdb641060d0 con 0x7fdb640ffc80 2023-07-14T11:05:06.643 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.637+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 15) v1 ==== 135659+0+0 (secure 0 0 0) 0x7fdb54011990 con 0x7fdb640ffc80 2023-07-14T11:05:06.643 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.637+0000 7fdb68964700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 0x7fdb4c0a3570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:06.643 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.637+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 2309+0+0 (secure 0 0 0) 0x7fdb540cab60 con 0x7fdb640ffc80 2023-07-14T11:05:06.643 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.637+0000 7fdb62ffd700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 0x7fdb4c0a3570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:06.643 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.638+0000 7fdb62ffd700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 0x7fdb4c0a3570 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fdb5800b5c0 tx=0x7fdb58005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:06.644 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.642+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdb54095b00 con 0x7fdb640ffc80 2023-07-14T11:05:06.838 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:06.835+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7fdb64060950 con 0x7fdb4c0a10c0 2023-07-14T11:05:07.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: osdmap e14: 2 total, 2 up, 2 in 2023-07-14T11:05:07.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:07.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:07.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:07.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:07.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: osdmap e14: 2 total, 2 up, 2 in 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:07.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:07.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: osdmap e14: 2 total, 2 up, 2 in 2023-07-14T11:05:07.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:07.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:07.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:07.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:07.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/3479993364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]': finished 2023-07-14T11:05:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: osdmap e15: 3 total, 2 up, 3 in 2023-07-14T11:05:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/3479993364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]': finished 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: osdmap e15: 3 total, 2 up, 3 in 2023-07-14T11:05:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/3479993364' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]: dispatch 2023-07-14T11:05:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a098e7e-81d5-4e1d-b613-ff144e76f342"}]': finished 2023-07-14T11:05:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: osdmap e15: 3 total, 2 up, 3 in 2023-07-14T11:05:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:11.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:10 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/3318998744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:10 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/3318998744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:10 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/3318998744' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:11 smithi130 ceph-mon[129405]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:11 smithi040 ceph-mon[97954]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:11 smithi045 ceph-mon[128791]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:13 smithi130 ceph-mon[129405]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:13 smithi040 ceph-mon[97954]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:13 smithi045 ceph-mon[128791]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:15.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:15 smithi130 ceph-mon[129405]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:15.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:15 smithi040 ceph-mon[97954]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:15.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:15 smithi045 ceph-mon[128791]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:16.510 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-07-14T11:05:16.511 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:16.511 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:16 smithi130 ceph-mon[129405]: Deploying daemon osd.2 on smithi130 2023-07-14T11:05:16.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-07-14T11:05:16.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:16.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:16 smithi040 ceph-mon[97954]: Deploying daemon osd.2 on smithi130 2023-07-14T11:05:16.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-07-14T11:05:16.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:16.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:16 smithi045 ceph-mon[128791]: Deploying daemon osd.2 on smithi130 2023-07-14T11:05:17.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:17 smithi130 ceph-mon[129405]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:17 smithi040 ceph-mon[97954]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:17 smithi045 ceph-mon[128791]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:19.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:19 smithi130 ceph-mon[129405]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:19 smithi040 ceph-mon[97954]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:19 smithi045 ceph-mon[128791]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:21.201 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:21.202 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:21 smithi130 ceph-mon[129405]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:21.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:21.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:21 smithi040 ceph-mon[97954]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:21.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:21.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:21.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:21 smithi045 ceph-mon[128791]: pgmap v72: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:22.751 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.749+0000 7fdb68964700 1 -- 172.21.15.130:0/2973524099 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fdb64060950 con 0x7fdb4c0a10c0 2023-07-14T11:05:22.752 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 2 on host 'smithi130' 2023-07-14T11:05:22.754 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.751+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 msgr2=0x7fdb4c0a3570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:22.754 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.751+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 0x7fdb4c0a3570 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fdb5800b5c0 tx=0x7fdb58005fb0 comp rx=0 tx=0).stop 2023-07-14T11:05:22.754 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.751+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 msgr2=0x7fdb64194da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:22.756 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.751+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7fdb5400b560 tx=0x7fdb5400b920 comp rx=0 tx=0).stop 2023-07-14T11:05:22.756 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.751+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 shutdown_connections 2023-07-14T11:05:22.756 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdb4c0a10c0 0x7fdb4c0a3570 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdb64100610 0x7fdb6419be20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdb640fea80 0x7fdb64194860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 --2- 172.21.15.130:0/2973524099 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdb640ffc80 0x7fdb64194da0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 >> 172.21.15.130:0/2973524099 conn(0x7fdb640fa0b0 msgr2=0x7fdb64103210 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 shutdown_connections 2023-07-14T11:05:22.757 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:05:22.752+0000 7fdb69966700 1 -- 172.21.15.130:0/2973524099 wait complete. 2023-07-14T11:05:23.179 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.2.service 2023-07-14T11:05:23.181 INFO:tasks.cephadm:Deploying osd.3 on smithi040 with /dev/vg_nvme/lv_3... 2023-07-14T11:05:23.181 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-07-14T11:05:23.217 INFO:journalctl@ceph.osd.2.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:05:23.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:23.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:23.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:23.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:23.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:23.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:23.922 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:05:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:24 smithi130 ceph-mon[129405]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:24 smithi130 ceph-mon[129405]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:24.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:05:23.999+0000 7f5afa813640 -1 osd.2 0 log_to_monitors true 2023-07-14T11:05:24.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:24 smithi040 ceph-mon[97954]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:24.496 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:24 smithi040 ceph-mon[97954]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:24.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:24 smithi045 ceph-mon[128791]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:24.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:24 smithi045 ceph-mon[128791]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-07-14T11:05:25.290 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:05:25.186+0000 7f5aefe8b700 -1 osd.2 0 waiting for initial osdmap 2023-07-14T11:05:25.290 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:05:25.189+0000 7f5ae9c7c700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: osdmap e16: 3 total, 2 up, 3 in 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:25 smithi130 ceph-mon[129405]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:25.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-07-14T11:05:25.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: osdmap e16: 3 total, 2 up, 3 in 2023-07-14T11:05:25.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:25.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:25 smithi040 ceph-mon[97954]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-07-14T11:05:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: osdmap e16: 3 total, 2 up, 3 in 2023-07-14T11:05:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: from='osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:05:25.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:25 smithi045 ceph-mon[128791]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:25.904 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-07-14T11:05:25.904 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-07-14T11:05:25.905 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10+0 records in 2023-07-14T11:05:25.905 INFO:teuthology.orchestra.run.smithi040.stderr:10+0 records out 2023-07-14T11:05:25.905 INFO:teuthology.orchestra.run.smithi040.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0426054 s, 246 MB/s 2023-07-14T11:05:25.905 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping successful for: 2023-07-14T11:05:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:05:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: osdmap e17: 3 total, 2 up, 3 in 2023-07-14T11:05:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi130 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 17168M 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:26.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:26.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.400 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi040:vg_nvme/lv_3 2023-07-14T11:05:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:05:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: osdmap e17: 3 total, 2 up, 3 in 2023-07-14T11:05:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi130 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 17168M 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:26.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:26.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:05:26.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: osdmap e17: 3 total, 2 up, 3 in 2023-07-14T11:05:26.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi130 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 17168M 2023-07-14T11:05:26.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:26.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:26.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:26.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:27.154 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:05:27.460 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] boot 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: osdmap e18: 3 total, 3 up, 3 in 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:27.461 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:27.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:05:27.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:05:27.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] boot 2023-07-14T11:05:27.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: osdmap e18: 3 total, 3 up, 3 in 2023-07-14T11:05:27.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:27.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:27.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:05:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:05:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: osd.2 [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] boot 2023-07-14T11:05:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: osdmap e18: 3 total, 3 up, 3 in 2023-07-14T11:05:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:05:27.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: pgmap v78: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2023-07-14T11:05:27.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:28.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:28.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:28 smithi130 ceph-mon[129405]: osdmap e19: 3 total, 3 up, 3 in 2023-07-14T11:05:28.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:28.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:28.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:28 smithi040 ceph-mon[97954]: osdmap e19: 3 total, 3 up, 3 in 2023-07-14T11:05:28.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:28.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:28.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:28 smithi045 ceph-mon[128791]: osdmap e19: 3 total, 3 up, 3 in 2023-07-14T11:05:28.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 -- 172.21.15.40:0/381084441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c074540 msgr2=0x7f5c0c0729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 --2- 172.21.15.40:0/381084441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c074540 0x7f5c0c0729a0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f5bfc0098f0 tx=0x7f5bfc009c00 comp rx=0 tx=0).stop 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 -- 172.21.15.40:0/381084441 shutdown_connections 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 --2- 172.21.15.40:0/381084441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c107a60 0x7f5c0c109ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 --2- 172.21.15.40:0/381084441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c074540 0x7f5c0c0729a0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:28.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 --2- 172.21.15.40:0/381084441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c0733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:28.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 -- 172.21.15.40:0/381084441 >> 172.21.15.40:0/381084441 conn(0x7f5c0c0fb1a0 msgr2=0x7f5c0c0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:28.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.813+0000 7f5c11edf700 1 -- 172.21.15.40:0/381084441 shutdown_connections 2023-07-14T11:05:28.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.814+0000 7f5c11edf700 1 -- 172.21.15.40:0/381084441 wait complete. 2023-07-14T11:05:28.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.814+0000 7f5c11edf700 1 Processor -- start 2023-07-14T11:05:28.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 -- start start 2023-07-14T11:05:28.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:28.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 0x7f5c0c198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:28.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c107a60 0x7f5c0c19f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c0c19f660 con 0x7f5c0c107a60 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c0c19f7a0 con 0x7f5c0c074540 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c11edf700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c0c19f930 con 0x7f5c0c072f70 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c0bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c107a60 0x7f5c0c19f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c0b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c0b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57182/0 (socket says 172.21.15.40:57182) 2023-07-14T11:05:28.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c0b7fe700 1 -- 172.21.15.40:0/674542838 learned_addr learned my addr 172.21.15.40:0/674542838 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.815+0000 7f5c0affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 0x7f5c0c198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 -- 172.21.15.40:0/674542838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 msgr2=0x7f5c0c198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 0x7f5c0c198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 -- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c107a60 msgr2=0x7f5c0c19f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c107a60 0x7f5c0c19f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 -- 172.21.15.40:0/674542838 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5bfc009520 con 0x7f5c0c072f70 2023-07-14T11:05:28.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0affd700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 0x7f5c0c198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:05:28.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c0b7fe700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f5bfc00c010 tx=0x7f5bfc00b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:28.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.816+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bfc01c070 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.817+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5bfc00bbf0 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.817+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c0c19fbb0 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.817+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5bfc004c30 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.817+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c0c19ffc0 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.818+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c0c106110 con 0x7f5c0c072f70 2023-07-14T11:05:28.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.820+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 15) v1 ==== 135659+0+0 (secure 0 0 0) 0x7f5bfc005040 con 0x7f5c0c072f70 2023-07-14T11:05:28.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.821+0000 7f5c08ff9700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 0x7f5bf40a35f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:28.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.821+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 3237+0+0 (secure 0 0 0) 0x7f5bfc0cc770 con 0x7f5c0c072f70 2023-07-14T11:05:28.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.822+0000 7f5c0affd700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 0x7f5bf40a35f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:28.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.822+0000 7f5c0affd700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 0x7f5bf40a35f0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5c00005fd0 tx=0x7f5c00009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:28.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:28.823+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5bfc097360 con 0x7f5c0c072f70 2023-07-14T11:05:29.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:29.042+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7f5c0c060950 con 0x7f5bf40a1140 2023-07-14T11:05:29.207 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:28 smithi040 sudo[113241]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:29.207 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:28 smithi040 sudo[113241]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:29.207 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:28 smithi040 sudo[113241]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:29.207 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:28 smithi040 sudo[113241]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:29.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:29.215+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5bfc097730 con 0x7f5c0c072f70 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: osdmap e20: 3 total, 3 up, 3 in 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 79 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:29.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:29.546 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:29 smithi040 sudo[113319]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:29.546 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:29 smithi040 sudo[113319]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:29.546 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:29 smithi040 sudo[113319]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:29.547 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:05:29 smithi040 sudo[113319]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:29.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:29.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: osdmap e20: 3 total, 3 up, 3 in 2023-07-14T11:05:29.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 79 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:29.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:29.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:29.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:29.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:05:29.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: osdmap e20: 3 total, 3 up, 3 in 2023-07-14T11:05:29.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 79 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:29.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:29.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:29.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:30.208 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:29 smithi045 sudo[136160]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:30.208 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:29 smithi045 sudo[136160]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:30.208 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:29 smithi045 sudo[136160]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:30.208 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:29 smithi045 sudo[136160]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:30.439 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:30 smithi040 ceph-mon[97954]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:30.439 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:30 smithi040 ceph-mon[97954]: osdmap e21: 3 total, 3 up, 3 in 2023-07-14T11:05:30.440 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:30 smithi040 ceph-mon[97954]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-07-14T11:05:30.440 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:30.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:30 smithi130 ceph-mon[129405]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:30.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:30 smithi130 ceph-mon[129405]: osdmap e21: 3 total, 3 up, 3 in 2023-07-14T11:05:30.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:30 smithi130 ceph-mon[129405]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-07-14T11:05:30.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:30.564 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:30 smithi045 sudo[136163]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:30.564 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:30 smithi045 sudo[136163]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:30.564 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:30 smithi045 sudo[136163]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:30.564 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:05:30 smithi045 sudo[136163]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:30.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:30 smithi045 ceph-mon[128791]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:30.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:30 smithi045 ceph-mon[128791]: osdmap e21: 3 total, 3 up, 3 in 2023-07-14T11:05:30.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:30 smithi045 ceph-mon[128791]: mgrmap e16: x(active, since 2m), standbys: z, y 2023-07-14T11:05:30.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:30.868 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137350]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:30.868 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137350]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:30.868 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137350]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:30.868 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137350]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:31.292 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:31.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137353]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:31.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:31.293 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:05:30 smithi130 sudo[137353]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113461]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113461]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113461]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:31.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113461]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:31 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137356]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137356]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:32.238 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137356]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:32.239 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137356]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113486]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113486]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113486]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:32.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 sudo[113486]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:32.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:31 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:32.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:32.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:31 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:32.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137359]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:32.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137359]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:32.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137359]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:32.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:32 smithi130 sudo[137359]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:33.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:32 smithi045 sudo[136166]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-07-14T11:05:33.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:32 smithi045 sudo[136166]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:33.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:32 smithi045 sudo[136166]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:33.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:32 smithi045 sudo[136166]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]: dispatch 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]': finished 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: osdmap e22: 4 total, 3 up, 4 in 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:33 smithi130 ceph-mon[129405]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]: dispatch 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]': finished 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: osdmap e22: 4 total, 3 up, 4 in 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:33 smithi040 ceph-mon[97954]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 sudo[136169]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-07-14T11:05:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 sudo[136169]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-07-14T11:05:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 sudo[136169]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-07-14T11:05:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 sudo[136169]: pam_unix(sudo:session): session closed for user root 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]: dispatch 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2098791927' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d065670e-a11a-4f71-a845-1059f16de6bd"}]': finished 2023-07-14T11:05:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: osdmap e22: 4 total, 3 up, 4 in 2023-07-14T11:05:33.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:33.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:05:33.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:05:33.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:05:33.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:33 smithi045 ceph-mon[128791]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:34.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:34 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:34.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:34 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2018364438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:34.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:34 smithi130 ceph-mon[129405]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:34 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:34 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2018364438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:34 smithi040 ceph-mon[97954]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:34 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-07-14T11:05:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:34 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2018364438' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:34 smithi045 ceph-mon[128791]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-07-14T11:05:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:35 smithi130 ceph-mon[129405]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:35.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:35 smithi040 ceph-mon[97954]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:35 smithi045 ceph-mon[128791]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:37.931 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:37 smithi040 ceph-mon[97954]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.4 KiB/s rd, 64 KiB/s wr, 6 op/s 2023-07-14T11:05:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:37 smithi130 ceph-mon[129405]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.4 KiB/s rd, 64 KiB/s wr, 6 op/s 2023-07-14T11:05:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:37 smithi045 ceph-mon[128791]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.4 KiB/s rd, 64 KiB/s wr, 6 op/s 2023-07-14T11:05:39.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:39 smithi040 ceph-mon[97954]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 43 KiB/s wr, 4 op/s 2023-07-14T11:05:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:39 smithi130 ceph-mon[129405]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 43 KiB/s wr, 4 op/s 2023-07-14T11:05:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:39 smithi045 ceph-mon[128791]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 43 KiB/s wr, 4 op/s 2023-07-14T11:05:40.906 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:40 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-07-14T11:05:40.906 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:40 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:40.907 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:40 smithi040 ceph-mon[97954]: Deploying daemon osd.3 on smithi040 2023-07-14T11:05:41.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:40 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-07-14T11:05:41.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:40 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:41.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:40 smithi130 ceph-mon[129405]: Deploying daemon osd.3 on smithi040 2023-07-14T11:05:41.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:40 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-07-14T11:05:41.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:40 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:41.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:40 smithi045 ceph-mon[128791]: Deploying daemon osd.3 on smithi040 2023-07-14T11:05:41.947 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:41 smithi040 ceph-mon[97954]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 32 KiB/s wr, 3 op/s 2023-07-14T11:05:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:41 smithi130 ceph-mon[129405]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 32 KiB/s wr, 3 op/s 2023-07-14T11:05:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:41 smithi045 ceph-mon[128791]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1.2 KiB/s rd, 32 KiB/s wr, 3 op/s 2023-07-14T11:05:43.724 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:43 smithi040 ceph-mon[97954]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1014 B/s rd, 26 KiB/s wr, 2 op/s 2023-07-14T11:05:44.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:43 smithi130 ceph-mon[129405]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1014 B/s rd, 26 KiB/s wr, 2 op/s 2023-07-14T11:05:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:43 smithi045 ceph-mon[128791]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 1014 B/s rd, 26 KiB/s wr, 2 op/s 2023-07-14T11:05:45.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:45 smithi130 ceph-mon[129405]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:45.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:45 smithi040 ceph-mon[97954]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:45.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:45 smithi045 ceph-mon[128791]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:47.699 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:47 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.699 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:47 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.699 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:47 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:47.700 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:47 smithi040 ceph-mon[97954]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:47 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:47 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:47 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:47 smithi130 ceph-mon[129405]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:47 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:47 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:47 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:47 smithi045 ceph-mon[128791]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail; 853 B/s rd, 22 KiB/s wr, 2 op/s 2023-07-14T11:05:49.916 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:49.916 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:49.916 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:49.917 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:49.917 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:49.917 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:50.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.353+0000 7f5c08ff9700 1 -- 172.21.15.40:0/674542838 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f5c0c060950 con 0x7f5bf40a1140 2023-07-14T11:05:50.357 INFO:teuthology.orchestra.run.smithi040.stdout:Created osd(s) 3 on host 'smithi040' 2023-07-14T11:05:50.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.356+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 msgr2=0x7f5bf40a35f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:50.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.356+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 0x7f5bf40a35f0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f5c00005fd0 tx=0x7f5c00009310 comp rx=0 tx=0).stop 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.356+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 msgr2=0x7f5c0c197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.356+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f5bfc00c010 tx=0x7f5bfc00b790 comp rx=0 tx=0).stop 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 shutdown_connections 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bf40a1140 0x7f5bf40a35f0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c0c107a60 0x7f5c0c19f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c0c072f70 0x7f5c0c197ad0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:50.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 --2- 172.21.15.40:0/674542838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c0c074540 0x7f5c0c198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:50.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 >> 172.21.15.40:0/674542838 conn(0x7f5c0c0fb1a0 msgr2=0x7f5c0c0fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:50.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 shutdown_connections 2023-07-14T11:05:50.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:05:50.357+0000 7f5c11edf700 1 -- 172.21.15.40:0/674542838 wait complete. 2023-07-14T11:05:50.853 DEBUG:teuthology.orchestra.run.smithi040:osd.3> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.3.service 2023-07-14T11:05:50.858 INFO:tasks.cephadm:Deploying osd.4 on smithi045 with /dev/vg_nvme/lv_3... 2023-07-14T11:05:50.858 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-07-14T11:05:50.935 INFO:journalctl@ceph.osd.3.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:05:51.434 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:05:51.657 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.658 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.658 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:51.658 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.658 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.658 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:51 smithi040 ceph-mon[97954]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:51.698 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.698 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.698 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:51.698 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.699 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.699 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:51 smithi045 ceph-mon[128791]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:51.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:05:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-07-14T11:05:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:51 smithi130 ceph-mon[129405]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:52.529 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-mon[97954]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-07-14T11:05:52.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-mon[97954]: osdmap e23: 4 total, 3 up, 4 in 2023-07-14T11:05:52.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:52.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-mon[97954]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-mon[97954]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.531 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:05:52.365+0000 7f7035734700 -1 osd.3 0 waiting for initial osdmap 2023-07-14T11:05:52.531 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:05:52 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:05:52.372+0000 7f702ed24700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:05:52.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:52 smithi130 ceph-mon[129405]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-07-14T11:05:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:52 smithi130 ceph-mon[129405]: osdmap e23: 4 total, 3 up, 4 in 2023-07-14T11:05:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:52 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:52 smithi130 ceph-mon[129405]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:52 smithi130 ceph-mon[129405]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:52 smithi045 ceph-mon[128791]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-07-14T11:05:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:52 smithi045 ceph-mon[128791]: osdmap e23: 4 total, 3 up, 4 in 2023-07-14T11:05:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:52 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:52 smithi045 ceph-mon[128791]: from='osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:52 smithi045 ceph-mon[128791]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:05:52.994 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-07-14T11:05:52.994 INFO:teuthology.orchestra.run.smithi045.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-07-14T11:05:52.994 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10+0 records in 2023-07-14T11:05:52.994 INFO:teuthology.orchestra.run.smithi045.stderr:10+0 records out 2023-07-14T11:05:52.995 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0132807 s, 790 MB/s 2023-07-14T11:05:52.995 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping successful for: 2023-07-14T11:05:53.383 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi045:vg_nvme/lv_3 2023-07-14T11:05:53.726 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:53 smithi040 ceph-mon[97954]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:05:53.727 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:53 smithi040 ceph-mon[97954]: osdmap e24: 4 total, 3 up, 4 in 2023-07-14T11:05:53.727 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.727 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.727 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:53 smithi040 ceph-mon[97954]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:53 smithi130 ceph-mon[129405]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:05:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:53 smithi130 ceph-mon[129405]: osdmap e24: 4 total, 3 up, 4 in 2023-07-14T11:05:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:53 smithi130 ceph-mon[129405]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:53 smithi045 ceph-mon[128791]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:05:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:53 smithi045 ceph-mon[128791]: osdmap e24: 4 total, 3 up, 4 in 2023-07-14T11:05:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:53 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:53 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:53 smithi045 ceph-mon[128791]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2023-07-14T11:05:53.945 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:05:54.499 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:05:54.499 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:05:54.499 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] boot 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: osdmap e25: 4 total, 4 up, 4 in 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi040 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.500 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 8636M 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.501 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:54 smithi045 ceph-mon[128791]: osdmap e26: 4 total, 4 up, 4 in 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] boot 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: osdmap e25: 4 total, 4 up, 4 in 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:54.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi040 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 8636M 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:54.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:54.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:54 smithi130 ceph-mon[129405]: osdmap e26: 4 total, 4 up, 4 in 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: osd.3 [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] boot 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: osdmap e25: 4 total, 4 up, 4 in 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi040 2023-07-14T11:05:54.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 8636M 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:54.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:05:54.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:05:54.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:54 smithi040 ceph-mon[97954]: osdmap e26: 4 total, 4 up, 4 in 2023-07-14T11:05:55.298 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 -- 172.21.15.45:0/2915640809 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e00ffc80 msgr2=0x7fa0e01000d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:55.298 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 --2- 172.21.15.45:0/2915640809 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e00ffc80 0x7fa0e01000d0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fa0d4009990 tx=0x7fa0d4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.298 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 -- 172.21.15.45:0/2915640809 shutdown_connections 2023-07-14T11:05:55.298 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 --2- 172.21.15.45:0/2915640809 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e0100610 0x7fa0e0106ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.298 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 --2- 172.21.15.45:0/2915640809 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00fea80 0x7fa0e00fee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 --2- 172.21.15.45:0/2915640809 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e00ffc80 0x7fa0e01000d0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 -- 172.21.15.45:0/2915640809 >> 172.21.15.45:0/2915640809 conn(0x7fa0e00fa0b0 msgr2=0x7fa0e00fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 -- 172.21.15.45:0/2915640809 shutdown_connections 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.293+0000 7fa0e6516700 1 -- 172.21.15.45:0/2915640809 wait complete. 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 Processor -- start 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 -- start start 2023-07-14T11:05:55.299 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e00fea80 0x7fa0e01958c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 0x7fa0e0195e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 0x7fa0e019ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e019d450 con 0x7fa0e00fea80 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e019d590 con 0x7fa0e0100610 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e6516700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0e019d720 con 0x7fa0e00ffc80 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0e4ab3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 0x7fa0e019ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:55.300 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0df7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 0x7fa0e0195e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0df7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 0x7fa0e0195e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.45:60130/0 (socket says 172.21.15.45:60130) 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0df7fe700 1 -- 172.21.15.45:0/1288909119 learned_addr learned my addr 172.21.15.45:0/1288909119 (peer_addr_for_me v2:172.21.15.45:0/0) 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.294+0000 7fa0dffff700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e00fea80 0x7fa0e01958c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 -- 172.21.15.45:0/1288909119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 msgr2=0x7fa0e0195e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 0x7fa0e0195e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 -- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e00fea80 msgr2=0x7fa0e01958c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e00fea80 0x7fa0e01958c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:05:55.301 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 -- 172.21.15.45:0/1288909119 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0d40095f0 con 0x7fa0e0100610 2023-07-14T11:05:55.302 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e4ab3700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 0x7fa0e019ce80 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fa0d800be20 tx=0x7fa0d8009ed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:55.302 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d800cc10 con 0x7fa0e0100610 2023-07-14T11:05:55.302 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa0e019da00 con 0x7fa0e0100610 2023-07-14T11:05:55.302 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa0d800cd70 con 0x7fa0e0100610 2023-07-14T11:05:55.302 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d80126c0 con 0x7fa0e0100610 2023-07-14T11:05:55.303 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.295+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa0e019df00 con 0x7fa0e0100610 2023-07-14T11:05:55.303 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.296+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa0e010a310 con 0x7fa0e0100610 2023-07-14T11:05:55.303 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.298+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa0d8014440 con 0x7fa0e0100610 2023-07-14T11:05:55.303 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.298+0000 7fa0dd7fa700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 0x7fa0c80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:05:55.304 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.298+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(27..27 src has 1..27) v4 ==== 3698+0+0 (secure 0 0 0) 0x7fa0d8006e40 con 0x7fa0e0100610 2023-07-14T11:05:55.304 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.299+0000 7fa0dffff700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 0x7fa0c80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:05:55.304 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.299+0000 7fa0dffff700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 0x7fa0c80a3600 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0005fd0 tx=0x7fa0d0005e50 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:05:55.305 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.302+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa0d8095dd0 con 0x7fa0e0100610 2023-07-14T11:05:55.514 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:05:55.511+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7fa0e010a530 con 0x7fa0c80a1150 2023-07-14T11:05:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:55 smithi130 ceph-mon[129405]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:05:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:55 smithi130 ceph-mon[129405]: osdmap e27: 4 total, 4 up, 4 in 2023-07-14T11:05:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:55 smithi040 ceph-mon[97954]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:05:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:55 smithi040 ceph-mon[97954]: osdmap e27: 4 total, 4 up, 4 in 2023-07-14T11:05:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:55 smithi045 ceph-mon[128791]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:05:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:55 smithi045 ceph-mon[128791]: osdmap e27: 4 total, 4 up, 4 in 2023-07-14T11:05:56.648 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:56 smithi045 ceph-mon[128791]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:56.649 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:56 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:56.649 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:56 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:56.649 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:56 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:56.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:56 smithi130 ceph-mon[129405]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:56.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:56 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:56.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:56 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:56.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:56 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:56.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:56 smithi040 ceph-mon[97954]: from='client.24218 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:05:56.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:56 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:05:56.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:56 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:05:56.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:56 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:05:57.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:57 smithi045 ceph-mon[128791]: osdmap e28: 4 total, 4 up, 4 in 2023-07-14T11:05:57.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:57 smithi045 ceph-mon[128791]: pgmap v103: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:57.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:57 smithi130 ceph-mon[129405]: osdmap e28: 4 total, 4 up, 4 in 2023-07-14T11:05:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:57 smithi130 ceph-mon[129405]: pgmap v103: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:57.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:57 smithi040 ceph-mon[97954]: osdmap e28: 4 total, 4 up, 4 in 2023-07-14T11:05:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:57 smithi040 ceph-mon[97954]: pgmap v103: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:58.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: osdmap e29: 4 total, 4 up, 4 in 2023-07-14T11:05:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/4094147838' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]': finished 2023-07-14T11:05:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: osdmap e30: 5 total, 4 up, 5 in 2023-07-14T11:05:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:58 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:05:58.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: osdmap e29: 4 total, 4 up, 4 in 2023-07-14T11:05:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/4094147838' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]': finished 2023-07-14T11:05:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: osdmap e30: 5 total, 4 up, 5 in 2023-07-14T11:05:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:58 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:05:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: osdmap e29: 4 total, 4 up, 4 in 2023-07-14T11:05:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/4094147838' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]: dispatch 2023-07-14T11:05:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6fe941ce-8407-48e2-ac92-bedcf87816b3"}]': finished 2023-07-14T11:05:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: osdmap e30: 5 total, 4 up, 5 in 2023-07-14T11:05:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:58 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:05:59.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:59 smithi130 ceph-mon[129405]: pgmap v106: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:05:59 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/2241306017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:59 smithi040 ceph-mon[97954]: pgmap v106: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:05:59 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/2241306017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:05:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:59 smithi045 ceph-mon[128791]: pgmap v106: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:05:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:05:59 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/2241306017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:01.889 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:01 smithi045 ceph-mon[128791]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:01 smithi130 ceph-mon[129405]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:01 smithi040 ceph-mon[97954]: pgmap v107: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:03.908 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:03 smithi045 ceph-mon[128791]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:03 smithi130 ceph-mon[129405]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:03 smithi040 ceph-mon[97954]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:05.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:05 smithi130 ceph-mon[129405]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:05.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-07-14T11:06:05.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:05 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:05.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:05 smithi130 ceph-mon[129405]: Deploying daemon osd.4 on smithi045 2023-07-14T11:06:05.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:05 smithi040 ceph-mon[97954]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:05.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-07-14T11:06:05.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:05 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:05.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:05 smithi040 ceph-mon[97954]: Deploying daemon osd.4 on smithi045 2023-07-14T11:06:05.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:05 smithi045 ceph-mon[128791]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:05.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-07-14T11:06:05.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:05 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:05.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:05 smithi045 ceph-mon[128791]: Deploying daemon osd.4 on smithi045 2023-07-14T11:06:08.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:07 smithi130 ceph-mon[129405]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:07 smithi040 ceph-mon[97954]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:07 smithi045 ceph-mon[128791]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:09 smithi130 ceph-mon[129405]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:09 smithi040 ceph-mon[97954]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:09 smithi045 ceph-mon[128791]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:11.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:11.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:11.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:12.482 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.480+0000 7fa0dd7fa700 1 -- 172.21.15.45:0/1288909119 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fa0e010a530 con 0x7fa0c80a1150 2023-07-14T11:06:12.483 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 4 on host 'smithi045' 2023-07-14T11:06:12.484 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 msgr2=0x7fa0c80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 0x7fa0c80a3600 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0005fd0 tx=0x7fa0d0005e50 comp rx=0 tx=0).stop 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 msgr2=0x7fa0e019ce80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 0x7fa0e019ce80 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fa0d800be20 tx=0x7fa0d8009ed0 comp rx=0 tx=0).stop 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 shutdown_connections 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c80a1150 0x7fa0c80a3600 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0e0100610 0x7fa0e019ce80 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:12.485 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0e00fea80 0x7fa0e01958c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:12.486 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 --2- 172.21.15.45:0/1288909119 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0e00ffc80 0x7fa0e0195e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:12.486 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 >> 172.21.15.45:0/1288909119 conn(0x7fa0e00fa0b0 msgr2=0x7fa0e0068050 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:06:12.486 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 shutdown_connections 2023-07-14T11:06:12.486 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:06:12.483+0000 7fa0e6516700 1 -- 172.21.15.45:0/1288909119 wait complete. 2023-07-14T11:06:12.936 DEBUG:teuthology.orchestra.run.smithi045:osd.4> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.4.service 2023-07-14T11:06:12.939 INFO:tasks.cephadm:Deploying osd.5 on smithi130 with /dev/vg_nvme/lv_3... 2023-07-14T11:06:12.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-07-14T11:06:12.984 INFO:journalctl@ceph.osd.4.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:06:13.514 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:06:13.564 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:06:13.239+0000 7fe2d9bf3640 -1 osd.4 0 log_to_monitors true 2023-07-14T11:06:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:13 smithi045 ceph-mon[128791]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:13.727 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.727 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:13.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:13.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:13.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:13 smithi130 ceph-mon[129405]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:13 smithi040 ceph-mon[97954]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-07-14T11:06:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:14 smithi130 ceph-mon[129405]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-07-14T11:06:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:14 smithi130 ceph-mon[129405]: osdmap e31: 5 total, 4 up, 5 in 2023-07-14T11:06:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:14 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:14 smithi130 ceph-mon[129405]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:14.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:14 smithi130 ceph-mon[129405]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:14.794 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:06:14.499+0000 7fe2cf26b700 -1 osd.4 0 waiting for initial osdmap 2023-07-14T11:06:14.794 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:06:14.504+0000 7fe2c7859700 -1 osd.4 32 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:06:14.794 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-mon[128791]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-07-14T11:06:14.795 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-mon[128791]: osdmap e31: 5 total, 4 up, 5 in 2023-07-14T11:06:14.795 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:14.795 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-mon[128791]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:14.795 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:14 smithi045 ceph-mon[128791]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:14 smithi040 ceph-mon[97954]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-07-14T11:06:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:14 smithi040 ceph-mon[97954]: osdmap e31: 5 total, 4 up, 5 in 2023-07-14T11:06:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:14 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:14 smithi040 ceph-mon[97954]: from='osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:14 smithi040 ceph-mon[97954]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:06:15.027 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-07-14T11:06:15.027 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-07-14T11:06:15.028 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-07-14T11:06:15.028 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-07-14T11:06:15.028 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110153 s, 952 MB/s 2023-07-14T11:06:15.028 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-07-14T11:06:15.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2023-07-14T11:06:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: osdmap e32: 5 total, 4 up, 5 in 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: osdmap e32: 5 total, 4 up, 5 in 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:06:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: osdmap e32: 5 total, 4 up, 5 in 2023-07-14T11:06:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:15.912 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi045 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 8586M 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] boot 2023-07-14T11:06:16.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: osdmap e33: 5 total, 5 up, 5 in 2023-07-14T11:06:16.722 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi045 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 8586M 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] boot 2023-07-14T11:06:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: osdmap e33: 5 total, 5 up, 5 in 2023-07-14T11:06:16.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:06:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:06:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi045 2023-07-14T11:06:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 8586M 2023-07-14T11:06:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: osd.4 [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] boot 2023-07-14T11:06:16.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: osdmap e33: 5 total, 5 up, 5 in 2023-07-14T11:06:16.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:06:17.213 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2754362147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38102f90 msgr2=0x7fea38105370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:17.213 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2754362147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38102f90 0x7fea38105370 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fea2c009990 tx=0x7fea2c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.213 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2754362147 shutdown_connections 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2754362147 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea381058b0 0x7fea38107cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2754362147 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100670 0x7fea38102a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2754362147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38102f90 0x7fea38105370 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2754362147 >> 172.21.15.130:0/2754362147 conn(0x7fea380fa0b0 msgr2=0x7fea380fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2754362147 shutdown_connections 2023-07-14T11:06:17.214 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2754362147 wait complete. 2023-07-14T11:06:17.215 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 Processor -- start 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.211+0000 7fea3e4b8700 1 -- start start 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38102f90 0x7fea380fe620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea381058b0 0x7fea380feb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea381997b0 con 0x7fea381058b0 2023-07-14T11:06:17.216 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea380ff0a0 con 0x7fea38102f90 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea380ff260 con 0x7fea38100670 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39422/0 (socket says 172.21.15.130:39422) 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 -- 172.21.15.130:0/2559067445 learned_addr learned my addr 172.21.15.130:0/2559067445 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 -- 172.21.15.130:0/2559067445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38102f90 msgr2=0x7fea380fe620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38102f90 0x7fea380fe620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 -- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea381058b0 msgr2=0x7fea380feb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:17.217 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea381058b0 0x7fea380feb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 -- 172.21.15.130:0/2559067445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea2c0095f0 con 0x7fea38100670 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea37fff700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fea2800b530 tx=0x7fea2800b8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea28011830 con 0x7fea38100670 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea381a4180 con 0x7fea38100670 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea28011e50 con 0x7fea38100670 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea2800f2e0 con 0x7fea38100670 2023-07-14T11:06:17.218 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.212+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea381a4590 con 0x7fea38100670 2023-07-14T11:06:17.219 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.213+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea3810b3f0 con 0x7fea38100670 2023-07-14T11:06:17.219 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.215+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fea28010580 con 0x7fea38100670 2023-07-14T11:06:17.219 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.215+0000 7fea357fa700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 0x7fea200a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:17.220 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.216+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(34..34 src has 1..34) v4 ==== 4101+0+0 (secure 0 0 0) 0x7fea280cb960 con 0x7fea38100670 2023-07-14T11:06:17.220 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.216+0000 7fea377fe700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 0x7fea200a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:17.220 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.216+0000 7fea377fe700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 0x7fea200a3600 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fea2c0095c0 tx=0x7fea2c009530 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:06:17.221 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.219+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fea28096120 con 0x7fea38100670 2023-07-14T11:06:17.416 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:17.414+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7fea38102250 con 0x7fea200a1150 2023-07-14T11:06:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:17 smithi130 ceph-mon[129405]: osdmap e34: 5 total, 5 up, 5 in 2023-07-14T11:06:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:17 smithi130 ceph-mon[129405]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:17 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:17 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:17 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:17 smithi040 ceph-mon[97954]: osdmap e34: 5 total, 5 up, 5 in 2023-07-14T11:06:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:17 smithi040 ceph-mon[97954]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:17 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:17 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:17 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:17 smithi045 ceph-mon[128791]: osdmap e34: 5 total, 5 up, 5 in 2023-07-14T11:06:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:17 smithi045 ceph-mon[128791]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:17 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:17 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:17 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:18.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:18 smithi130 ceph-mon[129405]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:18 smithi040 ceph-mon[97954]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:18 smithi045 ceph-mon[128791]: from='client.24244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:19 smithi130 ceph-mon[129405]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:19 smithi040 ceph-mon[97954]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:19 smithi045 ceph-mon[128791]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:20.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/44007951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]': finished 2023-07-14T11:06:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: osdmap e35: 6 total, 5 up, 6 in 2023-07-14T11:06:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:20.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:20 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/3247390733' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/44007951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]': finished 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: osdmap e35: 6 total, 5 up, 6 in 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:20.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:20 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/3247390733' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/44007951' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]: dispatch 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f6be8ae-e857-4175-b9ea-9da1eeabcfd0"}]': finished 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: osdmap e35: 6 total, 5 up, 6 in 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:20.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:20 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/3247390733' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:21.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:21 smithi130 ceph-mon[129405]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:21.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:21 smithi040 ceph-mon[97954]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:21.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:21 smithi045 ceph-mon[128791]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:23.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:23 smithi130 ceph-mon[129405]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:23 smithi040 ceph-mon[97954]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:23 smithi045 ceph-mon[128791]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:25.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:25 smithi130 ceph-mon[129405]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:25.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:25 smithi040 ceph-mon[97954]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:25.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:25 smithi045 ceph-mon[128791]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:26.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-07-14T11:06:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:26.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-07-14T11:06:26.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:26.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-07-14T11:06:26.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:27.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:27 smithi040 ceph-mon[97954]: Deploying daemon osd.5 on smithi130 2023-07-14T11:06:27.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:27 smithi040 ceph-mon[97954]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:27 smithi045 ceph-mon[128791]: Deploying daemon osd.5 on smithi130 2023-07-14T11:06:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:27 smithi045 ceph-mon[128791]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:27.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:27 smithi130 ceph-mon[129405]: Deploying daemon osd.5 on smithi130 2023-07-14T11:06:27.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:27 smithi130 ceph-mon[129405]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:29 smithi130 ceph-mon[129405]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:29 smithi040 ceph-mon[97954]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:29 smithi045 ceph-mon[128791]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:31 smithi130 ceph-mon[129405]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:31.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:31 smithi040 ceph-mon[97954]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:31 smithi045 ceph-mon[128791]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:33.481 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.478+0000 7fea357fa700 1 -- 172.21.15.130:0/2559067445 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fea38102250 con 0x7fea200a1150 2023-07-14T11:06:33.481 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 5 on host 'smithi130' 2023-07-14T11:06:33.483 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 msgr2=0x7fea200a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:33.483 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 0x7fea200a3600 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fea2c0095c0 tx=0x7fea2c009530 comp rx=0 tx=0).stop 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 msgr2=0x7fea380fffd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fea2800b530 tx=0x7fea2800b8f0 comp rx=0 tx=0).stop 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 shutdown_connections 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea200a1150 0x7fea200a3600 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea381058b0 0x7fea380feb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:33.484 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea38100670 0x7fea380fffd0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:33.485 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 --2- 172.21.15.130:0/2559067445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38102f90 0x7fea380fe620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:33.485 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 >> 172.21.15.130:0/2559067445 conn(0x7fea380fa0b0 msgr2=0x7fea38101b40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:06:33.485 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 shutdown_connections 2023-07-14T11:06:33.485 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:06:33.481+0000 7fea3e4b8700 1 -- 172.21.15.130:0/2559067445 wait complete. 2023-07-14T11:06:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:33.906 DEBUG:teuthology.orchestra.run.smithi130:osd.5> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.5.service 2023-07-14T11:06:33.908 INFO:tasks.cephadm:Deploying osd.6 on smithi040 with /dev/vg_nvme/lv_2... 2023-07-14T11:06:33.909 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-07-14T11:06:33.940 INFO:journalctl@ceph.osd.5.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:06:33.998 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:33.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.000 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.000 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:34.631 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:06:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:06:34.504+0000 7ff2eef02640 -1 osd.5 0 log_to_monitors true 2023-07-14T11:06:34.669 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:06:35.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:35 smithi130 ceph-mon[129405]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:35 smithi130 ceph-mon[129405]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:35 smithi130 ceph-mon[129405]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:35.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:35 smithi040 ceph-mon[97954]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:35 smithi040 ceph-mon[97954]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:35 smithi040 ceph-mon[97954]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:35.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:35 smithi045 ceph-mon[128791]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:35 smithi045 ceph-mon[128791]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-07-14T11:06:35.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:35 smithi045 ceph-mon[128791]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-07-14T11:06:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: osdmap e36: 6 total, 5 up, 6 in 2023-07-14T11:06:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:36.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:36.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.670 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-07-14T11:06:36.670 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-07-14T11:06:36.670 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10+0 records in 2023-07-14T11:06:36.670 INFO:teuthology.orchestra.run.smithi040.stderr:10+0 records out 2023-07-14T11:06:36.671 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0352815 s, 297 MB/s 2023-07-14T11:06:36.671 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping successful for: 2023-07-14T11:06:36.707 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-07-14T11:06:36.707 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: osdmap e36: 6 total, 5 up, 6 in 2023-07-14T11:06:36.707 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:36.707 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.707 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:36.708 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.792 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:06:36.302+0000 7ff2e457a700 -1 osd.5 0 waiting for initial osdmap 2023-07-14T11:06:36.793 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:06:36.310+0000 7ff2dd369700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:06:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-07-14T11:06:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: osdmap e36: 6 total, 5 up, 6 in 2023-07-14T11:06:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:36.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:06:36.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:37.152 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi040:vg_nvme/lv_2 2023-07-14T11:06:37.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi130 2023-07-14T11:06:37.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 8584M 2023-07-14T11:06:37.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: osdmap e37: 6 total, 5 up, 6 in 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] boot 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: osdmap e38: 6 total, 6 up, 6 in 2023-07-14T11:06:37.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:37 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi130 2023-07-14T11:06:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 8584M 2023-07-14T11:06:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:06:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: osdmap e37: 6 total, 5 up, 6 in 2023-07-14T11:06:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:37.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] boot 2023-07-14T11:06:37.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: osdmap e38: 6 total, 6 up, 6 in 2023-07-14T11:06:37.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:37 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi130 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 8584M 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: osdmap e37: 6 total, 5 up, 6 in 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2023-07-14T11:06:37.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: osd.5 [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] boot 2023-07-14T11:06:37.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: osdmap e38: 6 total, 6 up, 6 in 2023-07-14T11:06:37.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:37 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:06:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:06:38.451 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:38 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:06:38.452 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:38 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:06:38.452 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:38 smithi040 ceph-mon[97954]: osdmap e39: 6 total, 6 up, 6 in 2023-07-14T11:06:38.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:38 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:06:38.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:38 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:06:38.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:38 smithi045 ceph-mon[128791]: osdmap e39: 6 total, 6 up, 6 in 2023-07-14T11:06:38.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:38 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:06:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:38 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:06:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:38 smithi130 ceph-mon[129405]: osdmap e39: 6 total, 6 up, 6 in 2023-07-14T11:06:39.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/2293423585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8100d90 msgr2=0x7fa7c81011e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:39.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/2293423585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8100d90 0x7fa7c81011e0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc009990 tx=0x7fa7bc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/2293423585 shutdown_connections 2023-07-14T11:06:39.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/2293423585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c8101720 0x7fa7c8107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/2293423585 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/2293423585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8100d90 0x7fa7c81011e0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/2293423585 >> 172.21.15.40:0/2293423585 conn(0x7fa7c80fb1a0 msgr2=0x7fa7c80fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:06:39.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/2293423585 shutdown_connections 2023-07-14T11:06:39.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.382+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/2293423585 wait complete. 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.383+0000 7fa7ce2e3700 1 Processor -- start 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 -- start start 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c8100d90 0x7fa7c8072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 0x7fa7c8072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:39.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7c80735c0 con 0x7fa7c8101720 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7c8073730 con 0x7fa7c8100d90 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7ce2e3700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7c819f940 con 0x7fa7c80ffb90 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7c7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7c7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:42790/0 (socket says 172.21.15.40:42790) 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7c7fff700 1 -- 172.21.15.40:0/3450570735 learned_addr learned my addr 172.21.15.40:0/3450570735 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7c77fe700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c8100d90 0x7fa7c8072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:39.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.384+0000 7fa7cc880700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 0x7fa7c8072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 -- 172.21.15.40:0/3450570735 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c8100d90 msgr2=0x7fa7c8072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c8100d90 0x7fa7c8072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 -- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 msgr2=0x7fa7c8072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 0x7fa7c8072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 -- 172.21.15.40:0/3450570735 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7bc0095f0 con 0x7fa7c80ffb90 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7cc880700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 0x7fa7c8072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:06:39.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c7fff700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fa7b800d980 tx=0x7fa7b800dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:06:39.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b800f890 con 0x7fa7c80ffb90 2023-07-14T11:06:39.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa7b8010460 con 0x7fa7c80ffb90 2023-07-14T11:06:39.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7b80195c0 con 0x7fa7c80ffb90 2023-07-14T11:06:39.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.385+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7c819fc20 con 0x7fa7c80ffb90 2023-07-14T11:06:39.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.386+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7c81a0120 con 0x7fa7c80ffb90 2023-07-14T11:06:39.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.386+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa7c8107190 con 0x7fa7c80ffb90 2023-07-14T11:06:39.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.388+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa7b80105d0 con 0x7fa7c80ffb90 2023-07-14T11:06:39.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.390+0000 7fa7c57fa700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 0x7fa7b00a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:06:39.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.390+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4562+0+0 (secure 0 0 0) 0x7fa7b80cb920 con 0x7fa7c80ffb90 2023-07-14T11:06:39.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.390+0000 7fa7c77fe700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 0x7fa7b00a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:06:39.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.390+0000 7fa7c77fe700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 0x7fa7b00a3680 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc00b2e0 tx=0x7fa7bc005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:06:39.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.392+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa7b8095e70 con 0x7fa7c80ffb90 2023-07-14T11:06:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:39 smithi040 ceph-mon[97954]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:39 smithi040 ceph-mon[97954]: osdmap e40: 6 total, 6 up, 6 in 2023-07-14T11:06:39.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:06:39.607+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7fa7c8060950 con 0x7fa7b00a11d0 2023-07-14T11:06:39.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:39 smithi130 ceph-mon[129405]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:39 smithi130 ceph-mon[129405]: osdmap e40: 6 total, 6 up, 6 in 2023-07-14T11:06:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:39 smithi045 ceph-mon[128791]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:39 smithi045 ceph-mon[128791]: osdmap e40: 6 total, 6 up, 6 in 2023-07-14T11:06:40.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:40 smithi040 ceph-mon[97954]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:40.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:40 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:40.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:40 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:40.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:40 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:40 smithi130 ceph-mon[129405]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:40 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:40 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:40 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:40 smithi045 ceph-mon[128791]: from='client.24271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:06:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:40 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:06:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:40 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:06:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:40 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:41.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:41 smithi040 ceph-mon[97954]: osdmap e41: 6 total, 6 up, 6 in 2023-07-14T11:06:41.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:41 smithi040 ceph-mon[97954]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:41 smithi130 ceph-mon[129405]: osdmap e41: 6 total, 6 up, 6 in 2023-07-14T11:06:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:41 smithi130 ceph-mon[129405]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:41 smithi045 ceph-mon[128791]: osdmap e41: 6 total, 6 up, 6 in 2023-07-14T11:06:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:41 smithi045 ceph-mon[128791]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 161 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:42.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:42 smithi130 ceph-mon[129405]: osdmap e42: 6 total, 6 up, 6 in 2023-07-14T11:06:42.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:42 smithi040 ceph-mon[97954]: osdmap e42: 6 total, 6 up, 6 in 2023-07-14T11:06:42.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:42 smithi045 ceph-mon[128791]: osdmap e42: 6 total, 6 up, 6 in 2023-07-14T11:06:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:43 smithi130 ceph-mon[129405]: pgmap v140: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2023-07-14T11:06:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:43 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]: dispatch 2023-07-14T11:06:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:43 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]': finished 2023-07-14T11:06:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:43 smithi130 ceph-mon[129405]: osdmap e43: 7 total, 6 up, 7 in 2023-07-14T11:06:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:06:43.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:43 smithi040 ceph-mon[97954]: pgmap v140: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2023-07-14T11:06:43.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:43 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]: dispatch 2023-07-14T11:06:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:43 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]': finished 2023-07-14T11:06:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:43 smithi040 ceph-mon[97954]: osdmap e43: 7 total, 6 up, 7 in 2023-07-14T11:06:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:06:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:43 smithi045 ceph-mon[128791]: pgmap v140: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 134 KiB/s, 0 objects/s recovering 2023-07-14T11:06:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:43 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]: dispatch 2023-07-14T11:06:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:43 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2355167870' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04fdd37c-4112-429a-b73e-7ff7d0da5177"}]': finished 2023-07-14T11:06:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:43 smithi045 ceph-mon[128791]: osdmap e43: 7 total, 6 up, 7 in 2023-07-14T11:06:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:06:44.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:44 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2908588564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:44.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:44 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2908588564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:44.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:44 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2908588564' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:06:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:45 smithi130 ceph-mon[129405]: pgmap v142: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:45.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:45 smithi040 ceph-mon[97954]: pgmap v142: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:45.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:45 smithi045 ceph-mon[128791]: pgmap v142: 1 pgs: 1 peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:06:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:47 smithi130 ceph-mon[129405]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:06:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:47 smithi040 ceph-mon[97954]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:06:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:47 smithi045 ceph-mon[128791]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:06:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:49 smithi130 ceph-mon[129405]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:06:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:49 smithi040 ceph-mon[97954]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:06:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:49 smithi045 ceph-mon[128791]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:06:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:50 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-07-14T11:06:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:50 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:51.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:50 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-07-14T11:06:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:50 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:51.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:50 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-07-14T11:06:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:50 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:06:51.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:51 smithi040 ceph-mon[97954]: Deploying daemon osd.6 on smithi040 2023-07-14T11:06:51.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:51 smithi040 ceph-mon[97954]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:51 smithi130 ceph-mon[129405]: Deploying daemon osd.6 on smithi040 2023-07-14T11:06:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:51 smithi130 ceph-mon[129405]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:51 smithi045 ceph-mon[128791]: Deploying daemon osd.6 on smithi040 2023-07-14T11:06:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:51 smithi045 ceph-mon[128791]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:53.844 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:53 smithi040 ceph-mon[97954]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:53 smithi130 ceph-mon[129405]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:53 smithi045 ceph-mon[128791]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:55.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:55 smithi130 ceph-mon[129405]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:55.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:55 smithi040 ceph-mon[97954]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:55.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:55 smithi045 ceph-mon[128791]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:57.761 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:57 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.761 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:57 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:57.762 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:57 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.762 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:57 smithi040 ceph-mon[97954]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:57.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:57 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:57 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:57 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:57 smithi130 ceph-mon[129405]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:06:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:06:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:57 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:06:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:57 smithi045 ceph-mon[128791]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:06:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:06:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:06:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:00.275 INFO:teuthology.orchestra.run.smithi040.stdout:Created osd(s) 6 on host 'smithi040' 2023-07-14T11:07:00.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.272+0000 7fa7c57fa700 1 -- 172.21.15.40:0/3450570735 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fa7c8060950 con 0x7fa7b00a11d0 2023-07-14T11:07:00.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.275+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 msgr2=0x7fa7b00a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:00.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 0x7fa7b00a3680 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7fa7bc00b2e0 tx=0x7fa7bc005fb0 comp rx=0 tx=0).stop 2023-07-14T11:07:00.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 msgr2=0x7fa7c80742b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:00.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fa7b800d980 tx=0x7fa7b800dc90 comp rx=0 tx=0).stop 2023-07-14T11:07:00.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 shutdown_connections 2023-07-14T11:07:00.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa7b00a11d0 0x7fa7b00a3680 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7c8101720 0x7fa7c8072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7c80ffb90 0x7fa7c80742b0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 --2- 172.21.15.40:0/3450570735 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7c8100d90 0x7fa7c8072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 >> 172.21.15.40:0/3450570735 conn(0x7fa7c80fb1a0 msgr2=0x7fa7c81042d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 shutdown_connections 2023-07-14T11:07:00.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:00.276+0000 7fa7ce2e3700 1 -- 172.21.15.40:0/3450570735 wait complete. 2023-07-14T11:07:00.716 DEBUG:teuthology.orchestra.run.smithi040:osd.6> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.6.service 2023-07-14T11:07:00.721 INFO:tasks.cephadm:Deploying osd.7 on smithi045 with /dev/vg_nvme/lv_2... 2023-07-14T11:07:00.721 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-07-14T11:07:00.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:07:01.294 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:07:01.504 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.504 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.504 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:01 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:01.504 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:01 smithi045 ceph-mon[128791]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-07-14T11:07:01.504 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:01 smithi045 ceph-mon[128791]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:01.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:01 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:01.531 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:01 smithi040 ceph-mon[97954]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-07-14T11:07:01.531 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:01 smithi040 ceph-mon[97954]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:01.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:01.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:01 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:01.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:01 smithi130 ceph-mon[129405]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-07-14T11:07:01.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:01 smithi130 ceph-mon[129405]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:02.337 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:02 smithi045 ceph-mon[128791]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-07-14T11:07:02.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:02 smithi045 ceph-mon[128791]: osdmap e44: 7 total, 6 up, 7 in 2023-07-14T11:07:02.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:02 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:02.338 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:02 smithi045 ceph-mon[128791]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:07:02.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:02 smithi130 ceph-mon[129405]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-07-14T11:07:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:02 smithi130 ceph-mon[129405]: osdmap e44: 7 total, 6 up, 7 in 2023-07-14T11:07:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:02 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:02.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:02 smithi130 ceph-mon[129405]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:07:02.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-mon[97954]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-07-14T11:07:02.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-mon[97954]: osdmap e44: 7 total, 6 up, 7 in 2023-07-14T11:07:02.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:02.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-mon[97954]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:07:02.546 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:07:02.263+0000 7f14a095f700 -1 osd.6 0 waiting for initial osdmap 2023-07-14T11:07:02.547 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:07:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:07:02.274+0000 7f149a750700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:07:02.790 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-07-14T11:07:02.791 INFO:teuthology.orchestra.run.smithi045.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-07-14T11:07:02.791 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10+0 records in 2023-07-14T11:07:02.791 INFO:teuthology.orchestra.run.smithi045.stderr:10+0 records out 2023-07-14T11:07:02.791 INFO:teuthology.orchestra.run.smithi045.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00806794 s, 1.3 GB/s 2023-07-14T11:07:02.791 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping successful for: 2023-07-14T11:07:03.250 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi045:vg_nvme/lv_2 2023-07-14T11:07:03.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:03 smithi040 ceph-mon[97954]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:07:03.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:03 smithi040 ceph-mon[97954]: osdmap e45: 7 total, 6 up, 7 in 2023-07-14T11:07:03.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:03 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:03 smithi040 ceph-mon[97954]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:03.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:03 smithi130 ceph-mon[129405]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:07:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:03 smithi130 ceph-mon[129405]: osdmap e45: 7 total, 6 up, 7 in 2023-07-14T11:07:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:03 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:03 smithi130 ceph-mon[129405]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:03.563 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:03 smithi045 ceph-mon[128791]: from='osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469]' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:07:03.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:03 smithi045 ceph-mon[128791]: osdmap e45: 7 total, 6 up, 7 in 2023-07-14T11:07:03.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:03 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:03 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:03.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:03 smithi045 ceph-mon[128791]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2023-07-14T11:07:03.812 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] boot 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: osdmap e46: 7 total, 7 up, 7 in 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:04.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi040 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 5757M 2023-07-14T11:07:04.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:04.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:04.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:04 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] boot 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: osdmap e46: 7 total, 7 up, 7 in 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi040 2023-07-14T11:07:04.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 5757M 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:04.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:04.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:04 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: osd.6 [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] boot 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: osdmap e46: 7 total, 7 up, 7 in 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi040 2023-07-14T11:07:04.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 5757M 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:04.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:04.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:04.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:04 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:05.113 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 -- 172.21.15.45:0/57042003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 msgr2=0x7fb794106ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:05.113 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 --2- 172.21.15.45:0/57042003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb794106ed0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fb78c00b0a0 tx=0x7fb78c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 -- 172.21.15.45:0/57042003 shutdown_connections 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 --2- 172.21.15.45:0/57042003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb794106ed0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 --2- 172.21.15.45:0/57042003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 0x7fb7940fee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 --2- 172.21.15.45:0/57042003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 0x7fb7941000d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 -- 172.21.15.45:0/57042003 >> 172.21.15.45:0/57042003 conn(0x7fb7940fa0b0 msgr2=0x7fb7940fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 -- 172.21.15.45:0/57042003 shutdown_connections 2023-07-14T11:07:05.114 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.111+0000 7fb79b297700 1 -- 172.21.15.45:0/57042003 wait complete. 2023-07-14T11:07:05.116 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 Processor -- start 2023-07-14T11:07:05.116 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 -- start start 2023-07-14T11:07:05.116 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 0x7fb7941029c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 0x7fb794101060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb794101d40 con 0x7fb7940fea80 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb794101eb0 con 0x7fb794100610 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb79b297700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb79419f9f0 con 0x7fb7940ffc80 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb799834700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb799834700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.45:54130/0 (socket says 172.21.15.45:54130) 2023-07-14T11:07:05.117 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb799834700 1 -- 172.21.15.45:0/2743278177 learned_addr learned my addr 172.21.15.45:0/2743278177 (peer_addr_for_me v2:172.21.15.45:0/0) 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.112+0000 7fb798832700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 0x7fb794101060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799033700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 0x7fb7941029c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 -- 172.21.15.45:0/2743278177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 msgr2=0x7fb794101060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 0x7fb794101060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 -- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 msgr2=0x7fb7941029c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 0x7fb7941029c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:05.118 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 -- 172.21.15.45:0/2743278177 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb78c009d00 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb799834700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fb78c0062a0 tx=0x7fb78c004d00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb78c00e040 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb78c00cec0 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb79419fc20 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb78c01baf0 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.113+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb7941a00e0 con 0x7fb794100610 2023-07-14T11:07:05.119 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.114+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb79410a250 con 0x7fb794100610 2023-07-14T11:07:05.120 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.115+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb78c01bc50 con 0x7fb794100610 2023-07-14T11:07:05.120 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.116+0000 7fb78a7fc700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 0x7fb7800a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:05.120 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.116+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(47..47 src has 1..47) v4 ==== 4965+0+0 (secure 0 0 0) 0x7fb78c0cde00 con 0x7fb794100610 2023-07-14T11:07:05.120 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.116+0000 7fb799033700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 0x7fb7800a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:05.121 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.117+0000 7fb799033700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 0x7fb7800a3680 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fb7940ffae0 tx=0x7fb79000b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:05.121 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.119+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb78c0982f0 con 0x7fb794100610 2023-07-14T11:07:05.326 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:05.324+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7fb794068700 con 0x7fb7800a11d0 2023-07-14T11:07:05.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:05 smithi130 ceph-mon[129405]: osdmap e47: 7 total, 7 up, 7 in 2023-07-14T11:07:05.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:05 smithi130 ceph-mon[129405]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:05.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:05 smithi040 ceph-mon[97954]: osdmap e47: 7 total, 7 up, 7 in 2023-07-14T11:07:05.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:05 smithi040 ceph-mon[97954]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:05.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:05 smithi045 ceph-mon[128791]: osdmap e47: 7 total, 7 up, 7 in 2023-07-14T11:07:05.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:05 smithi045 ceph-mon[128791]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:06.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:06 smithi045 ceph-mon[128791]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:06.507 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:06.508 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:06.508 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:06.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:06 smithi040 ceph-mon[97954]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:06.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:06.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:06.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:06.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:06 smithi130 ceph-mon[129405]: from='client.24284 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:07.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:07 smithi045 ceph-mon[128791]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:07.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:07 smithi130 ceph-mon[129405]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:07.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:07 smithi040 ceph-mon[97954]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:08.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/3468410094' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:08 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:08 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]': finished 2023-07-14T11:07:08.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:08 smithi045 ceph-mon[128791]: osdmap e48: 8 total, 7 up, 8 in 2023-07-14T11:07:08.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:08 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:08.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/3468410094' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:08 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:08 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]': finished 2023-07-14T11:07:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:08 smithi130 ceph-mon[129405]: osdmap e48: 8 total, 7 up, 8 in 2023-07-14T11:07:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:08 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/3468410094' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:08 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]: dispatch 2023-07-14T11:07:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:08 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50a6c695-d861-4a14-8473-06d9c6d991d6"}]': finished 2023-07-14T11:07:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:08 smithi040 ceph-mon[97954]: osdmap e48: 8 total, 7 up, 8 in 2023-07-14T11:07:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:08 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:09.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/2535491074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:09.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:09 smithi045 ceph-mon[128791]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:09.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/2535491074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:09 smithi130 ceph-mon[129405]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/2535491074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:09 smithi040 ceph-mon[97954]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:11 smithi045 ceph-mon[128791]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:11 smithi130 ceph-mon[129405]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:11 smithi040 ceph-mon[97954]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:13 smithi130 ceph-mon[129405]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:13 smithi040 ceph-mon[97954]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:14.050 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:13 smithi045 ceph-mon[128791]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:15.387 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-07-14T11:07:15.387 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:15.387 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:15 smithi045 ceph-mon[128791]: Deploying daemon osd.7 on smithi045 2023-07-14T11:07:15.387 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:15 smithi045 ceph-mon[128791]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-07-14T11:07:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:15 smithi130 ceph-mon[129405]: Deploying daemon osd.7 on smithi045 2023-07-14T11:07:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:15 smithi130 ceph-mon[129405]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-07-14T11:07:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:15 smithi040 ceph-mon[97954]: Deploying daemon osd.7 on smithi045 2023-07-14T11:07:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:15 smithi040 ceph-mon[97954]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:17 smithi130 ceph-mon[129405]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:17 smithi040 ceph-mon[97954]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:17 smithi045 ceph-mon[128791]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:20.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:19 smithi130 ceph-mon[129405]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:20.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:19 smithi040 ceph-mon[97954]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:20.301 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:19 smithi045 ceph-mon[128791]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:20.301 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.301 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:20.301 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:21.842 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 7 on host 'smithi045' 2023-07-14T11:07:21.843 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.840+0000 7fb78a7fc700 1 -- 172.21.15.45:0/2743278177 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fb794068700 con 0x7fb7800a11d0 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 msgr2=0x7fb7800a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 0x7fb7800a3680 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fb7940ffae0 tx=0x7fb79000b410 comp rx=0 tx=0).stop 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 msgr2=0x7fb7941015a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fb78c0062a0 tx=0x7fb78c004d00 comp rx=0 tx=0).stop 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 shutdown_connections 2023-07-14T11:07:21.845 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb7800a11d0 0x7fb7800a3680 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb794100610 0x7fb7941015a0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7940fea80 0x7fb7941029c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 --2- 172.21.15.45:0/2743278177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb7940ffc80 0x7fb794101060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 >> 172.21.15.45:0/2743278177 conn(0x7fb7940fa0b0 msgr2=0x7fb794067ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 shutdown_connections 2023-07-14T11:07:21.846 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:07:21.843+0000 7fb79b297700 1 -- 172.21.15.45:0/2743278177 wait complete. 2023-07-14T11:07:22.030 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.031 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:22.032 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:22.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:22.298 DEBUG:teuthology.orchestra.run.smithi045:osd.7> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.7.service 2023-07-14T11:07:22.300 INFO:tasks.cephadm:Deploying osd.8 on smithi130 with /dev/vg_nvme/lv_2... 2023-07-14T11:07:22.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-07-14T11:07:22.330 INFO:journalctl@ceph.osd.7.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:07:22.864 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:07:23.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:22 smithi045 ceph-mon[128791]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:23.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:22 smithi045 ceph-mon[128791]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:23.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:07:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:07:22.948+0000 7f7d89335640 -1 osd.7 0 log_to_monitors true 2023-07-14T11:07:23.249 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:22 smithi130 ceph-mon[129405]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:23.250 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:22 smithi130 ceph-mon[129405]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:23.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:22 smithi040 ceph-mon[97954]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:23.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:22 smithi040 ceph-mon[97954]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-07-14T11:07:24.278 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:24.278 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-07-14T11:07:24.278 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: osdmap e49: 8 total, 7 up, 8 in 2023-07-14T11:07:24.278 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:24.279 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.279 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-mon[128791]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.279 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:07:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:07:23.991+0000 7f7d7e9ad700 -1 osd.7 0 waiting for initial osdmap 2023-07-14T11:07:24.279 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:07:24.000+0000 7f7d7a7a2700 -1 osd.7 50 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:07:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-07-14T11:07:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: osdmap e49: 8 total, 7 up, 8 in 2023-07-14T11:07:24.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:24.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:23 smithi130 ceph-mon[129405]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-07-14T11:07:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: osdmap e49: 8 total, 7 up, 8 in 2023-07-14T11:07:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: from='osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:23 smithi040 ceph-mon[97954]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:07:24.346 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-07-14T11:07:24.346 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-07-14T11:07:24.346 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-07-14T11:07:24.346 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-07-14T11:07:24.347 INFO:teuthology.orchestra.run.smithi130.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0154893 s, 677 MB/s 2023-07-14T11:07:24.347 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-07-14T11:07:24.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2023-07-14T11:07:25.259 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:07:25.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: osdmap e50: 8 total, 7 up, 8 in 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] boot 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: osdmap e51: 8 total, 8 up, 8 in 2023-07-14T11:07:25.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:25.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:25.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:24 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: osdmap e50: 8 total, 7 up, 8 in 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] boot 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: osdmap e51: 8 total, 8 up, 8 in 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:25.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:24 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:07:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: osdmap e50: 8 total, 7 up, 8 in 2023-07-14T11:07:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: osd.7 [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] boot 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: osdmap e51: 8 total, 8 up, 8 in 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:25.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:25.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:25.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:25.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:24 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi045 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 5724M 2023-07-14T11:07:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:25 smithi130 ceph-mon[129405]: osdmap e52: 8 total, 8 up, 8 in 2023-07-14T11:07:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:07:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:07:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi045 2023-07-14T11:07:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 5724M 2023-07-14T11:07:26.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:25 smithi040 ceph-mon[97954]: osdmap e52: 8 total, 8 up, 8 in 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi045 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 5724M 2023-07-14T11:07:26.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:25 smithi045 ceph-mon[128791]: osdmap e52: 8 total, 8 up, 8 in 2023-07-14T11:07:26.428 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/4263872635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 msgr2=0x7fa0dc0ff3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/4263872635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc0ff3c0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fa0d0009990 tx=0x7fa0d0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/4263872635 shutdown_connections 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/4263872635 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc100680 0x7fa0dc106f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/4263872635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc0ff3c0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/4263872635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc0ffcd0 0x7fa0dc100140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.429 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/4263872635 >> 172.21.15.130:0/4263872635 conn(0x7fa0dc0f9860 msgr2=0x7fa0dc0fbc90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:26.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/4263872635 shutdown_connections 2023-07-14T11:07:26.430 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.426+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/4263872635 wait complete. 2023-07-14T11:07:26.431 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 Processor -- start 2023-07-14T11:07:26.431 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 -- start start 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc0ffcd0 0x7fa0dc195df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 0x7fa0dc19ce70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0dc19d440 con 0x7fa0dc100680 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0dc19d580 con 0x7fa0dc0ffcd0 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0e0ae8700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa0dc19d710 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.432 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39530/0 (socket says 172.21.15.130:39530) 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 -- 172.21.15.130:0/2315148781 learned_addr learned my addr 172.21.15.130:0/2315148781 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0d9d9b700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc0ffcd0 0x7fa0dc195df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0dad9d700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 0x7fa0dc19ce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 -- 172.21.15.130:0/2315148781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc0ffcd0 msgr2=0x7fa0dc195df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc0ffcd0 0x7fa0dc195df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 -- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 msgr2=0x7fa0dc19ce70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:26.433 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 0x7fa0dc19ce70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0da59c700 1 -- 172.21.15.130:0/2315148781 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa0d00095f0 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.427+0000 7fa0dad9d700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 0x7fa0dc19ce70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0da59c700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fa0d000c010 tx=0x7fa0d000f700 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d001d070 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa0dc19d990 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa0d000fbe0 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa0d0017650 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.434 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.428+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa0dc19de00 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.435 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.429+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa0dc10a2f0 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.435 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.430+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa0d0022470 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.435 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.430+0000 7fa0cb7fe700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 0x7fa0c40a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:26.436 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.430+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(53..53 src has 1..53) v4 ==== 5426+0+0 (secure 0 0 0) 0x7fa0d00cdac0 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.436 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.431+0000 7fa0d9d9b700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 0x7fa0c40a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:26.436 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.431+0000 7fa0d9d9b700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 0x7fa0c40a3600 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fa0dc0fece0 tx=0x7fa0cc00a3b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:26.436 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.434+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa0d0097de0 con 0x7fa0dc0fefb0 2023-07-14T11:07:26.630 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:26.628+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7fa0dc0fe2e0 con 0x7fa0c40a1150 2023-07-14T11:07:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: osdmap e53: 8 total, 8 up, 8 in 2023-07-14T11:07:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 215 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:27.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:27.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:27.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:27.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: osdmap e53: 8 total, 8 up, 8 in 2023-07-14T11:07:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 215 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:27.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: osdmap e53: 8 total, 8 up, 8 in 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 215 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:28.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:28 smithi130 ceph-mon[129405]: osdmap e54: 8 total, 8 up, 8 in 2023-07-14T11:07:28.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:28 smithi040 ceph-mon[97954]: osdmap e54: 8 total, 8 up, 8 in 2023-07-14T11:07:28.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:28 smithi045 ceph-mon[128791]: osdmap e54: 8 total, 8 up, 8 in 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: osdmap e55: 8 total, 8 up, 8 in 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/2273066155' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]': finished 2023-07-14T11:07:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: osdmap e56: 9 total, 8 up, 9 in 2023-07-14T11:07:29.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:29.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: osdmap e55: 8 total, 8 up, 8 in 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/2273066155' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]': finished 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: osdmap e56: 9 total, 8 up, 9 in 2023-07-14T11:07:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: osdmap e55: 8 total, 8 up, 8 in 2023-07-14T11:07:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/2273066155' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]: dispatch 2023-07-14T11:07:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2fcfa29f-64c0-450c-ba0d-2a74a38550bb"}]': finished 2023-07-14T11:07:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: osdmap e56: 9 total, 8 up, 9 in 2023-07-14T11:07:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:30.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:30 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/3953167814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:30.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:30 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/3953167814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:30.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:30 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/3953167814' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:31 smithi130 ceph-mon[129405]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:31.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:31 smithi040 ceph-mon[97954]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:31 smithi045 ceph-mon[128791]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:33.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:33 smithi130 ceph-mon[129405]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:33 smithi040 ceph-mon[97954]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:33 smithi045 ceph-mon[128791]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:35.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:35 smithi045 ceph-mon[128791]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:35 smithi130 ceph-mon[129405]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:35 smithi040 ceph-mon[97954]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:36.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-07-14T11:07:36.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:36.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:36 smithi130 ceph-mon[129405]: Deploying daemon osd.8 on smithi130 2023-07-14T11:07:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-07-14T11:07:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:36.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:36 smithi045 ceph-mon[128791]: Deploying daemon osd.8 on smithi130 2023-07-14T11:07:36.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.8"}]: dispatch 2023-07-14T11:07:36.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:36.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:36 smithi040 ceph-mon[97954]: Deploying daemon osd.8 on smithi130 2023-07-14T11:07:37.522 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:37 smithi130 ceph-mon[129405]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:37.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:37 smithi045 ceph-mon[128791]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:37 smithi040 ceph-mon[97954]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:39.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:39 smithi130 ceph-mon[129405]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:39 smithi040 ceph-mon[97954]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:39 smithi045 ceph-mon[128791]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:41.130 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:41 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:41.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:41 smithi130 ceph-mon[129405]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:41.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:41 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:41.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:41 smithi040 ceph-mon[97954]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:41.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:41.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:41 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:41.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:41 smithi045 ceph-mon[128791]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:42.695 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 8 on host 'smithi130' 2023-07-14T11:07:42.695 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.692+0000 7fa0cb7fe700 1 -- 172.21.15.130:0/2315148781 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fa0dc0fe2e0 con 0x7fa0c40a1150 2023-07-14T11:07:42.697 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 msgr2=0x7fa0c40a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:42.697 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 0x7fa0c40a3600 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fa0dc0fece0 tx=0x7fa0cc00a3b0 comp rx=0 tx=0).stop 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 msgr2=0x7fa0dc1958b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7fa0d000c010 tx=0x7fa0d000f700 comp rx=0 tx=0).stop 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 shutdown_connections 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa0c40a1150 0x7fa0c40a3600 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa0dc100680 0x7fa0dc19ce70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa0dc0fefb0 0x7fa0dc1958b0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:42.698 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 --2- 172.21.15.130:0/2315148781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa0dc0ffcd0 0x7fa0dc195df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:42.699 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 >> 172.21.15.130:0/2315148781 conn(0x7fa0dc0f9860 msgr2=0x7fa0dc0fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:42.699 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 shutdown_connections 2023-07-14T11:07:42.699 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:07:42.695+0000 7fa0e0ae8700 1 -- 172.21.15.130:0/2315148781 wait complete. 2023-07-14T11:07:43.142 DEBUG:teuthology.orchestra.run.smithi130:osd.8> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.8.service 2023-07-14T11:07:43.145 INFO:tasks.cephadm:Deploying osd.9 on smithi040 with /dev/vg_nvme/lv_1... 2023-07-14T11:07:43.145 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-07-14T11:07:43.175 INFO:journalctl@ceph.osd.8.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:07:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:43.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:07:43.887 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:07:44.242 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:07:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:07:43.959+0000 7f51acaf5640 -1 osd.8 0 log_to_monitors true 2023-07-14T11:07:44.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:44 smithi130 ceph-mon[129405]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:44.501 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:44 smithi130 ceph-mon[129405]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:44.673 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:44 smithi040 ceph-mon[97954]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:44.673 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:44 smithi040 ceph-mon[97954]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:44.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:44 smithi045 ceph-mon[128791]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:44.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:44 smithi045 ceph-mon[128791]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]: dispatch 2023-07-14T11:07:45.785 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-07-14T11:07:45.785 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-07-14T11:07:45.785 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10+0 records in 2023-07-14T11:07:45.785 INFO:teuthology.orchestra.run.smithi040.stderr:10+0 records out 2023-07-14T11:07:45.786 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.029856 s, 351 MB/s 2023-07-14T11:07:45.786 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping successful for: 2023-07-14T11:07:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-07-14T11:07:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: osdmap e57: 9 total, 8 up, 9 in 2023-07-14T11:07:45.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:45.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-mon[129405]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:45.793 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:07:45.426+0000 7f51a216d700 -1 osd.8 0 waiting for initial osdmap 2023-07-14T11:07:45.793 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:07:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:07:45.435+0000 7f519c75f700 -1 osd.8 58 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:07:45.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-07-14T11:07:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: osdmap e57: 9 total, 8 up, 9 in 2023-07-14T11:07:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:45 smithi040 ceph-mon[97954]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["8"]}]': finished 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: osdmap e57: 9 total, 8 up, 9 in 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: from='osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487]' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: from='osd.8 ' entity='osd.8' cmd=[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:07:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:45 smithi045 ceph-mon[128791]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 216 MiB used, 715 GiB / 715 GiB avail 2023-07-14T11:07:46.267 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi040:vg_nvme/lv_1 2023-07-14T11:07:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:07:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: osdmap e58: 9 total, 8 up, 9 in 2023-07-14T11:07:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi130 2023-07-14T11:07:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 5722M 2023-07-14T11:07:46.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:46.548 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:46.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] boot 2023-07-14T11:07:46.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: osdmap e59: 9 total, 9 up, 9 in 2023-07-14T11:07:46.549 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:07:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: osdmap e58: 9 total, 8 up, 9 in 2023-07-14T11:07:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi130 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 5722M 2023-07-14T11:07:46.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] boot 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: osdmap e59: 9 total, 9 up, 9 in 2023-07-14T11:07:46.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='osd.8 ' entity='osd.8' cmd='[{"prefix": "osd crush create-or-move", "id": 8, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:07:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: osdmap e58: 9 total, 8 up, 9 in 2023-07-14T11:07:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi130 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 5722M 2023-07-14T11:07:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: osd.8 [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] boot 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: osdmap e59: 9 total, 9 up, 9 in 2023-07-14T11:07:46.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:07:47.004 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:07:47.675 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:47 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:07:47.675 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:47 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:07:47.675 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:47 smithi040 ceph-mon[97954]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 243 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:47.675 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:47 smithi040 ceph-mon[97954]: osdmap e60: 9 total, 9 up, 9 in 2023-07-14T11:07:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:47 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:07:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:47 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:07:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:47 smithi130 ceph-mon[129405]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 243 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:47 smithi130 ceph-mon[129405]: osdmap e60: 9 total, 9 up, 9 in 2023-07-14T11:07:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:47 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:07:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:47 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:07:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:47 smithi045 ceph-mon[128791]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 243 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:47 smithi045 ceph-mon[128791]: osdmap e60: 9 total, 9 up, 9 in 2023-07-14T11:07:48.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 -- 172.21.15.40:0/2846401361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b00101720 msgr2=0x7f8b00107fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:48.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/2846401361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b00101720 0x7f8b00107fe0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f8af800b0a0 tx=0x7f8af800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 -- 172.21.15.40:0/2846401361 shutdown_connections 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/2846401361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b00101720 0x7f8b00107fe0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/2846401361 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b000ffb90 0x7f8b000fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/2846401361 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00100d90 0x7f8b001011e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 -- 172.21.15.40:0/2846401361 >> 172.21.15.40:0/2846401361 conn(0x7f8b000fb1a0 msgr2=0x7f8b000fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 -- 172.21.15.40:0/2846401361 shutdown_connections 2023-07-14T11:07:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.607+0000 7f8b07e2f700 1 -- 172.21.15.40:0/2846401361 wait complete. 2023-07-14T11:07:48.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.608+0000 7f8b07e2f700 1 Processor -- start 2023-07-14T11:07:48.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 -- start start 2023-07-14T11:07:48.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b00100d90 0x7f8b00195eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00101720 0x7f8b0019cf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b0019d500 con 0x7f8b000ffb90 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b0019d640 con 0x7f8b00100d90 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b07e2f700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b0019d7d0 con 0x7f8b00101720 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.609+0000 7f8b05bcb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:48.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37412/0 (socket says 172.21.15.40:37412) 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 -- 172.21.15.40:0/350313740 learned_addr learned my addr 172.21.15.40:0/350313740 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b053ca700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b00100d90 0x7f8b00195eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b063cc700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00101720 0x7f8b0019cf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 -- 172.21.15.40:0/350313740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00101720 msgr2=0x7f8b0019cf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00101720 0x7f8b0019cf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 -- 172.21.15.40:0/350313740 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b00100d90 msgr2=0x7f8b00195eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:07:48.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b00100d90 0x7f8b00195eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 -- 172.21.15.40:0/350313740 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8af8009d00 con 0x7f8b000ffb90 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b05bcb700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f8afc00e9a0 tx=0x7f8afc00ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8afc009790 con 0x7f8b000ffb90 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b0019dab0 con 0x7f8b000ffb90 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.610+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8afc0044e0 con 0x7f8b000ffb90 2023-07-14T11:07:48.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.611+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8afc003d20 con 0x7f8b000ffb90 2023-07-14T11:07:48.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.611+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b0019e000 con 0x7f8b000ffb90 2023-07-14T11:07:48.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.612+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b00107130 con 0x7f8b000ffb90 2023-07-14T11:07:48.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.614+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8afc003e80 con 0x7f8b000ffb90 2023-07-14T11:07:48.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.615+0000 7f8af6ffd700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 0x7f8aec0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:07:48.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.615+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(61..61 src has 1..61) v4 ==== 5858+0+0 (secure 0 0 0) 0x7f8afc014070 con 0x7f8b000ffb90 2023-07-14T11:07:48.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.616+0000 7f8b053ca700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 0x7f8aec0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:07:48.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.616+0000 7f8b053ca700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 0x7f8aec0a3680 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f8af0009c60 tx=0x7f8af0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:07:48.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.619+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8afc096270 con 0x7f8b000ffb90 2023-07-14T11:07:48.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:07:48.835+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f8b00060950 con 0x7f8aec0a11d0 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: osdmap e61: 9 total, 9 up, 9 in 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:49.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:49.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:49 smithi130 ceph-mon[129405]: osdmap e62: 9 total, 9 up, 9 in 2023-07-14T11:07:49.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: osdmap e61: 9 total, 9 up, 9 in 2023-07-14T11:07:49.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:49.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:49 smithi040 ceph-mon[97954]: osdmap e62: 9 total, 9 up, 9 in 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: osdmap e61: 9 total, 9 up, 9 in 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: from='client.14367 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:07:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:07:49.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:49 smithi045 ceph-mon[128791]: osdmap e62: 9 total, 9 up, 9 in 2023-07-14T11:07:51.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:51 smithi130 ceph-mon[129405]: osdmap e63: 9 total, 9 up, 9 in 2023-07-14T11:07:51.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:51 smithi130 ceph-mon[129405]: pgmap v195: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:07:51.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:51 smithi040 ceph-mon[97954]: osdmap e63: 9 total, 9 up, 9 in 2023-07-14T11:07:51.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:51 smithi040 ceph-mon[97954]: pgmap v195: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:07:51.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:51 smithi045 ceph-mon[128791]: osdmap e63: 9 total, 9 up, 9 in 2023-07-14T11:07:51.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:51 smithi045 ceph-mon[128791]: pgmap v195: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-07-14T11:07:52.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/196401110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:52 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:52 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]': finished 2023-07-14T11:07:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:52 smithi130 ceph-mon[129405]: osdmap e64: 10 total, 9 up, 10 in 2023-07-14T11:07:52.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:52 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:07:52.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/196401110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:52 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:52 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]': finished 2023-07-14T11:07:52.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:52 smithi040 ceph-mon[97954]: osdmap e64: 10 total, 9 up, 10 in 2023-07-14T11:07:52.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:52 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:07:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/196401110' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:52 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]: dispatch 2023-07-14T11:07:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:52 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f"}]': finished 2023-07-14T11:07:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:52 smithi045 ceph-mon[128791]: osdmap e64: 10 total, 9 up, 10 in 2023-07-14T11:07:52.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:52 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:07:53.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:53 smithi130 ceph-mon[129405]: pgmap v197: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 138 KiB/s, 0 objects/s recovering 2023-07-14T11:07:53.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:53 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3513352121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:53.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:53 smithi040 ceph-mon[97954]: pgmap v197: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 138 KiB/s, 0 objects/s recovering 2023-07-14T11:07:53.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:53 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3513352121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:53 smithi045 ceph-mon[128791]: pgmap v197: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 138 KiB/s, 0 objects/s recovering 2023-07-14T11:07:53.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:53 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3513352121' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:07:55.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:55 smithi045 ceph-mon[128791]: pgmap v198: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-07-14T11:07:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:55 smithi130 ceph-mon[129405]: pgmap v198: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-07-14T11:07:55.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:55 smithi040 ceph-mon[97954]: pgmap v198: 1 pgs: 1 peering; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-07-14T11:07:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:57 smithi130 ceph-mon[129405]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-07-14T11:07:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:57 smithi040 ceph-mon[97954]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-07-14T11:07:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:57 smithi045 ceph-mon[128791]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail; 79 KiB/s, 0 objects/s recovering 2023-07-14T11:08:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:59 smithi130 ceph-mon[129405]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-07-14T11:08:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:07:59 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:59 smithi040 ceph-mon[97954]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-07-14T11:08:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:07:59 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:59 smithi045 ceph-mon[128791]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.9"}]: dispatch 2023-07-14T11:08:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:07:59 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:00.752 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:00 smithi040 ceph-mon[97954]: Deploying daemon osd.9 on smithi040 2023-07-14T11:08:01.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:00 smithi130 ceph-mon[129405]: Deploying daemon osd.9 on smithi040 2023-07-14T11:08:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:00 smithi045 ceph-mon[128791]: Deploying daemon osd.9 on smithi040 2023-07-14T11:08:01.966 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:01 smithi040 ceph-mon[97954]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:01 smithi130 ceph-mon[129405]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:01 smithi045 ceph-mon[128791]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:03.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:03 smithi040 ceph-mon[97954]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:03 smithi130 ceph-mon[129405]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:03 smithi045 ceph-mon[128791]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:05.466 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:05 smithi040 ceph-mon[97954]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:05.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:05 smithi045 ceph-mon[128791]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:05.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:05 smithi130 ceph-mon[129405]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:06.789 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.789 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.789 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:06.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:06.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:06.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:07.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:07 smithi130 ceph-mon[129405]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:07.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:07 smithi040 ceph-mon[97954]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:07.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:07 smithi045 ceph-mon[128791]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.951+0000 7f8af6ffd700 1 -- 172.21.15.40:0/350313740 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f8b00060950 con 0x7f8aec0a11d0 2023-07-14T11:08:08.954 INFO:teuthology.orchestra.run.smithi040.stdout:Created osd(s) 9 on host 'smithi040' 2023-07-14T11:08:08.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.953+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 msgr2=0x7f8aec0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:08.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.953+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 0x7f8aec0a3680 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f8af0009c60 tx=0x7f8af0009260 comp rx=0 tx=0).stop 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.953+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 msgr2=0x7f8b00195970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.953+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f8afc00e9a0 tx=0x7f8afc00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 shutdown_connections 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8aec0a11d0 0x7f8aec0a3680 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b00101720 0x7f8b0019cf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b000ffb90 0x7f8b00195970 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 --2- 172.21.15.40:0/350313740 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b00100d90 0x7f8b00195eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 >> 172.21.15.40:0/350313740 conn(0x7f8b000fb1a0 msgr2=0x7f8b00104270 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 shutdown_connections 2023-07-14T11:08:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:08.954+0000 7f8b07e2f700 1 -- 172.21.15.40:0/350313740 wait complete. 2023-07-14T11:08:09.427 DEBUG:teuthology.orchestra.run.smithi040:osd.9> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.9.service 2023-07-14T11:08:09.431 INFO:tasks.cephadm:Deploying osd.10 on smithi045 with /dev/vg_nvme/lv_1... 2023-07-14T11:08:09.431 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-07-14T11:08:09.518 INFO:journalctl@ceph.osd.9.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:08:10.002 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:08:10.165 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:10.165 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.166 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.166 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.166 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.166 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:10.166 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:09 smithi040 ceph-mon[97954]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:10.229 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:10.230 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:09 smithi045 ceph-mon[128791]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:10.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:09 smithi130 ceph-mon[129405]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]: dispatch 2023-07-14T11:08:11.327 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-07-14T11:08:11.327 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: osdmap e65: 10 total, 9 up, 10 in 2023-07-14T11:08:11.328 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:11.328 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.328 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.328 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-mon[97954]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:11.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-07-14T11:08:11.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: osdmap e65: 10 total, 9 up, 10 in 2023-07-14T11:08:11.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:11.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:11 smithi045 ceph-mon[128791]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:11.595 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-07-14T11:08:11.595 INFO:teuthology.orchestra.run.smithi045.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-07-14T11:08:11.595 INFO:teuthology.orchestra.run.smithi045.stderr: stderr: 10+0 records in 2023-07-14T11:08:11.596 INFO:teuthology.orchestra.run.smithi045.stderr:10+0 records out 2023-07-14T11:08:11.596 INFO:teuthology.orchestra.run.smithi045.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00830895 s, 1.3 GB/s 2023-07-14T11:08:11.596 INFO:teuthology.orchestra.run.smithi045.stderr:--> Zapping successful for: 2023-07-14T11:08:11.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["9"]}]': finished 2023-07-14T11:08:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: osdmap e65: 10 total, 9 up, 10 in 2023-07-14T11:08:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: from='osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249]' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: from='osd.9 ' entity='osd.9' cmd=[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]: dispatch 2023-07-14T11:08:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:11 smithi130 ceph-mon[129405]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 244 MiB used, 804 GiB / 805 GiB avail 2023-07-14T11:08:11.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:08:11.323+0000 7f378a7ba700 -1 osd.9 0 waiting for initial osdmap 2023-07-14T11:08:11.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:08:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:08:11.337+0000 7f37825a7700 -1 osd.9 66 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:08:12.061 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi045:vg_nvme/lv_1 2023-07-14T11:08:12.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: osdmap e66: 10 total, 9 up, 10 in 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:12.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:12 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.623 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:08:12.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: osdmap e66: 10 total, 9 up, 10 in 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:12.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:12 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='osd.9 ' entity='osd.9' cmd='[{"prefix": "osd crush create-or-move", "id": 9, "weight":0.0873, "args": ["host=smithi040", "root=default"]}]': finished 2023-07-14T11:08:12.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: osdmap e66: 10 total, 9 up, 10 in 2023-07-14T11:08:12.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:12.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:12.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:12.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:12.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:12 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:08:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:08:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi040 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 4318M 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] boot 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: osdmap e67: 10 total, 10 up, 10 in 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:13.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:13.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:13 smithi045 ceph-mon[128791]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:13.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:08:13.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:08:13.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi040 2023-07-14T11:08:13.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 4318M 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] boot 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: osdmap e67: 10 total, 10 up, 10 in 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:13.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:13.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:13 smithi130 ceph-mon[129405]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:13.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi040 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 4318M 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: osd.9 [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] boot 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: osdmap e67: 10 total, 10 up, 10 in 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:08:13.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:13.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:13 smithi040 ceph-mon[97954]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:14.004 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- 172.21.15.45:0/394559317 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 msgr2=0x7f4884106ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:14.004 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 --2- 172.21.15.45:0/394559317 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f4884106ed0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f487c00b0a0 tx=0x7f487c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- 172.21.15.45:0/394559317 shutdown_connections 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 --2- 172.21.15.45:0/394559317 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f4884106ed0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 --2- 172.21.15.45:0/394559317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48840fea80 0x7f48840fee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 --2- 172.21.15.45:0/394559317 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f48841000d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- 172.21.15.45:0/394559317 >> 172.21.15.45:0/394559317 conn(0x7f48840fa0b0 msgr2=0x7f48840fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:14.005 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- 172.21.15.45:0/394559317 shutdown_connections 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- 172.21.15.45:0/394559317 wait complete. 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 Processor -- start 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.001+0000 7f488a81e700 1 -- start start 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48840fea80 0x7f4884195940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f4884195e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f488419d4d0 con 0x7f48840fea80 2023-07-14T11:08:14.006 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f488419d610 con 0x7f4884100610 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f488a81e700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f488419d7a0 con 0x7f48840ffc80 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.45:54278/0 (socket says 172.21.15.45:54278) 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 -- 172.21.15.45:0/99620375 learned_addr learned my addr 172.21.15.45:0/99620375 (peer_addr_for_me v2:172.21.15.45:0/0) 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f48837fe700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f4884195e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 -- 172.21.15.45:0/99620375 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 msgr2=0x7f4884195e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:14.007 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f4884195e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.008 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 -- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48840fea80 msgr2=0x7f4884195940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:14.008 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48840fea80 0x7f4884195940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:14.008 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 -- 172.21.15.45:0/99620375 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f487c009d00 con 0x7f4884100610 2023-07-14T11:08:14.008 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f48837fe700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f4884195e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:08:14.008 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.002+0000 7f4888dbb700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f487c0062a0 tx=0x7f487c004db0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:14.009 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.003+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f487c00e040 con 0x7f4884100610 2023-07-14T11:08:14.009 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.003+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f487c01db50 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.003+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f488419da20 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.003+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f487c01bc30 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.003+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f488419de90 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.004+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4884106ad0 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.005+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f487c01d690 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.006+0000 7f48817fa700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 0x7f486c0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.006+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(68..68 src has 1..68) v4 ==== 6261+0+0 (secure 0 0 0) 0x7f487c0ce3a0 con 0x7f4884100610 2023-07-14T11:08:14.010 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.006+0000 7f4883fff700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 0x7f486c0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:14.011 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.007+0000 7f4883fff700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 0x7f486c0a3600 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f48840ffae0 tx=0x7f4874009470 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:14.011 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.010+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f487c098380 con 0x7f4884100610 2023-07-14T11:08:14.216 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:14.214+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7f4884060950 con 0x7f486c0a1150 2023-07-14T11:08:14.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:14 smithi130 ceph-mon[129405]: osdmap e68: 10 total, 10 up, 10 in 2023-07-14T11:08:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:14 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:14 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:14.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:14 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:14.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:14 smithi040 ceph-mon[97954]: osdmap e68: 10 total, 10 up, 10 in 2023-07-14T11:08:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:14 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:14 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:14.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:14 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:14.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:14 smithi045 ceph-mon[128791]: osdmap e68: 10 total, 10 up, 10 in 2023-07-14T11:08:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:14 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:14 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:14 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:15.613 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:15 smithi045 ceph-mon[128791]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:15.614 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:15 smithi045 ceph-mon[128791]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:15 smithi130 ceph-mon[129405]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:15 smithi130 ceph-mon[129405]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:15 smithi040 ceph-mon[97954]: from='client.24353 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:15 smithi040 ceph-mon[97954]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:17.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:16 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/2503078241' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:16 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:16 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]': finished 2023-07-14T11:08:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:16 smithi130 ceph-mon[129405]: osdmap e69: 11 total, 10 up, 11 in 2023-07-14T11:08:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:17.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:16 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/2503078241' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:16 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:16 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]': finished 2023-07-14T11:08:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:16 smithi040 ceph-mon[97954]: osdmap e69: 11 total, 10 up, 11 in 2023-07-14T11:08:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:16 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/2503078241' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:16 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]: dispatch 2023-07-14T11:08:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:16 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8fb75cd9-7dc5-4825-b957-ebe7b33c49c7"}]': finished 2023-07-14T11:08:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:16 smithi045 ceph-mon[128791]: osdmap e69: 11 total, 10 up, 11 in 2023-07-14T11:08:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:17 smithi130 ceph-mon[129405]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:17 smithi130 ceph-mon[129405]: from='client.? 172.21.15.45:0/2395981337' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:17 smithi040 ceph-mon[97954]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:17 smithi040 ceph-mon[97954]: from='client.? 172.21.15.45:0/2395981337' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:17 smithi045 ceph-mon[128791]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 271 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:17 smithi045 ceph-mon[128791]: from='client.? 172.21.15.45:0/2395981337' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:19 smithi130 ceph-mon[129405]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:19 smithi040 ceph-mon[97954]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:19 smithi045 ceph-mon[128791]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:21.777 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:21 smithi045 ceph-mon[128791]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:21 smithi130 ceph-mon[129405]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:21 smithi040 ceph-mon[97954]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:23.939 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:23 smithi045 ceph-mon[128791]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:23.939 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-07-14T11:08:23.939 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:23 smithi130 ceph-mon[129405]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-07-14T11:08:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:23 smithi040 ceph-mon[97954]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.10"}]: dispatch 2023-07-14T11:08:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:24.970 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:24 smithi045 ceph-mon[128791]: Deploying daemon osd.10 on smithi045 2023-07-14T11:08:25.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:24 smithi130 ceph-mon[129405]: Deploying daemon osd.10 on smithi045 2023-07-14T11:08:25.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:24 smithi040 ceph-mon[97954]: Deploying daemon osd.10 on smithi045 2023-07-14T11:08:26.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:25 smithi130 ceph-mon[129405]: pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:26.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:25 smithi040 ceph-mon[97954]: pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:26.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:25 smithi045 ceph-mon[128791]: pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:27 smithi130 ceph-mon[129405]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:27 smithi040 ceph-mon[97954]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:27 smithi045 ceph-mon[128791]: pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:28.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:29.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:29.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:29 smithi045 ceph-mon[128791]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:29 smithi130 ceph-mon[129405]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:30.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:29 smithi040 ceph-mon[97954]: pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:30.530 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.527+0000 7f48817fa700 1 -- 172.21.15.45:0/99620375 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7f4884060950 con 0x7f486c0a1150 2023-07-14T11:08:30.530 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 10 on host 'smithi045' 2023-07-14T11:08:30.532 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 msgr2=0x7f486c0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 0x7f486c0a3600 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f48840ffae0 tx=0x7f4874009470 comp rx=0 tx=0).stop 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 msgr2=0x7f488419cf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f487c0062a0 tx=0x7f487c004db0 comp rx=0 tx=0).stop 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 shutdown_connections 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f486c0a1150 0x7f486c0a3600 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4884100610 0x7f488419cf00 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:30.533 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48840fea80 0x7f4884195940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:30.534 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 --2- 172.21.15.45:0/99620375 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48840ffc80 0x7f4884195e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:30.534 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 >> 172.21.15.45:0/99620375 conn(0x7f48840fa0b0 msgr2=0x7f48840fc220 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:30.534 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 shutdown_connections 2023-07-14T11:08:30.534 INFO:teuthology.orchestra.run.smithi045.stderr:2023-07-14T11:08:30.530+0000 7f488a81e700 1 -- 172.21.15.45:0/99620375 wait complete. 2023-07-14T11:08:30.919 DEBUG:teuthology.orchestra.run.smithi045:osd.10> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.10.service 2023-07-14T11:08:30.922 INFO:tasks.cephadm:Deploying osd.11 on smithi130 with /dev/vg_nvme/lv_1... 2023-07-14T11:08:30.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-07-14T11:08:30.954 INFO:journalctl@ceph.osd.10.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:08:31.489 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:08:31.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:31.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:31.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:31.545 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:31 smithi130 ceph-mon[129405]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:31.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:31 smithi040 ceph-mon[97954]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:31.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:31.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:31.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:31.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:31.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-mon[128791]: pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:31.965 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:08:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:08:31.583+0000 7f161f339640 -1 osd.10 0 log_to_monitors true 2023-07-14T11:08:32.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:32 smithi130 ceph-mon[129405]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:32.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:32 smithi130 ceph-mon[129405]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:32.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:32 smithi040 ceph-mon[97954]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:32.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:32 smithi040 ceph-mon[97954]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:32.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:32 smithi045 ceph-mon[128791]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:32.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:32 smithi045 ceph-mon[128791]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]: dispatch 2023-07-14T11:08:33.020 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-07-14T11:08:33.020 INFO:teuthology.orchestra.run.smithi130.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-07-14T11:08:33.021 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10+0 records in 2023-07-14T11:08:33.021 INFO:teuthology.orchestra.run.smithi130.stderr:10+0 records out 2023-07-14T11:08:33.021 INFO:teuthology.orchestra.run.smithi130.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112289 s, 934 MB/s 2023-07-14T11:08:33.021 INFO:teuthology.orchestra.run.smithi130.stderr:--> Zapping successful for: 2023-07-14T11:08:33.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2023-07-14T11:08:33.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: osdmap e70: 11 total, 10 up, 11 in 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:33.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:33.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:33.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-07-14T11:08:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: osdmap e70: 11 total, 10 up, 11 in 2023-07-14T11:08:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:33.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:33.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.813 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:08:33.502+0000 7f16149b1700 -1 osd.10 0 waiting for initial osdmap 2023-07-14T11:08:33.814 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:08:33.514+0000 7f160d7a0700 -1 osd.10 71 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:08:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["10"]}]': finished 2023-07-14T11:08:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: osdmap e70: 11 total, 10 up, 11 in 2023-07-14T11:08:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425]' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='osd.10 ' entity='osd.10' cmd=[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 272 MiB used, 894 GiB / 894 GiB avail 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:33.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:33.982 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:08:34.735 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi045 2023-07-14T11:08:34.735 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 4293M 2023-07-14T11:08:34.735 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:08:34.735 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: osdmap e71: 11 total, 10 up, 11 in 2023-07-14T11:08:34.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] boot 2023-07-14T11:08:34.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: osdmap e72: 11 total, 11 up, 11 in 2023-07-14T11:08:34.736 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:34 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi045 2023-07-14T11:08:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 4293M 2023-07-14T11:08:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:08:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: osdmap e71: 11 total, 10 up, 11 in 2023-07-14T11:08:34.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] boot 2023-07-14T11:08:34.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: osdmap e72: 11 total, 11 up, 11 in 2023-07-14T11:08:34.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:34 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi045 2023-07-14T11:08:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 4293M 2023-07-14T11:08:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: from='osd.10 ' entity='osd.10' cmd='[{"prefix": "osd crush create-or-move", "id": 10, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2023-07-14T11:08:34.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: osdmap e71: 11 total, 10 up, 11 in 2023-07-14T11:08:34.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:34.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: osd.10 [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] boot 2023-07-14T11:08:34.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: osdmap e72: 11 total, 11 up, 11 in 2023-07-14T11:08:34.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:34 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:08:35.204 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 -- 172.21.15.130:0/1328791541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d00fea80 msgr2=0x7ff8d00fee90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:35.204 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/1328791541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d00fea80 0x7ff8d00fee90 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7ff8c0009990 tx=0x7ff8c0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.204 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 -- 172.21.15.130:0/1328791541 shutdown_connections 2023-07-14T11:08:35.204 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/1328791541 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d0100610 0x7ff8d0106ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.205 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/1328791541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d00fea80 0x7ff8d00fee90 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.205 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/1328791541 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 0x7ff8d01000d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.205 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 -- 172.21.15.130:0/1328791541 >> 172.21.15.130:0/1328791541 conn(0x7ff8d00fa0b0 msgr2=0x7ff8d00fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:35.205 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 -- 172.21.15.130:0/1328791541 shutdown_connections 2023-07-14T11:08:35.205 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.201+0000 7ff8d51bc700 1 -- 172.21.15.130:0/1328791541 wait complete. 2023-07-14T11:08:35.207 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 Processor -- start 2023-07-14T11:08:35.207 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 -- start start 2023-07-14T11:08:35.207 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 0x7ff8d0068b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 0x7ff8d019a5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8d019d3d0 con 0x7ff8d00fea80 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8d019d540 con 0x7ff8d00ffc80 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.202+0000 7ff8d51bc700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8d019d700 con 0x7ff8d0100610 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39682/0 (socket says 172.21.15.130:39682) 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 -- 172.21.15.130:0/3531473548 learned_addr learned my addr 172.21.15.130:0/3531473548 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8ce59c700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 0x7ff8d019a5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8ced9d700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 0x7ff8d0068b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 -- 172.21.15.130:0/3531473548 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 msgr2=0x7ff8d019a5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 0x7ff8d019a5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 -- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 msgr2=0x7ff8d0068b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:35.209 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 0x7ff8d0068b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 -- 172.21.15.130:0/3531473548 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8c00095f0 con 0x7ff8d0100610 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8ced9d700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 0x7ff8d0068b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8cf59e700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7ff8c8007ee0 tx=0x7ff8c800d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c8017070 con 0x7ff8d0100610 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8c800f040 con 0x7ff8d0100610 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8d019d9e0 con 0x7ff8d0100610 2023-07-14T11:08:35.210 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8c8005740 con 0x7ff8d0100610 2023-07-14T11:08:35.211 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.203+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8d019de90 con 0x7ff8d0100610 2023-07-14T11:08:35.211 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.204+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8d010a240 con 0x7ff8d0100610 2023-07-14T11:08:35.211 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.205+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7ff8c800dc60 con 0x7ff8d0100610 2023-07-14T11:08:35.212 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.206+0000 7ff8b7fff700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 0x7ff8b80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:35.212 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.206+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(72..72 src has 1..72) v4 ==== 6727+0+0 (secure 0 0 0) 0x7ff8c80cc130 con 0x7ff8d0100610 2023-07-14T11:08:35.212 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.206+0000 7ff8ced9d700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 0x7ff8b80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:35.212 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.207+0000 7ff8ced9d700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 0x7ff8b80a3600 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ff8c000b2e0 tx=0x7ff8c0005e80 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:35.212 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.209+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff8c8095eb0 con 0x7ff8d0100610 2023-07-14T11:08:35.406 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:35.403+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}) v1 -- 0x7ff8d0060950 con 0x7ff8b80a1150 2023-07-14T11:08:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: osdmap e73: 11 total, 11 up, 11 in 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:35 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:35.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: osdmap e73: 11 total, 11 up, 11 in 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:35 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: osdmap e73: 11 total, 11 up, 11 in 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-07-14T11:08:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:35 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:36.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:36 smithi130 ceph-mon[129405]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:36.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:36 smithi130 ceph-mon[129405]: osdmap e74: 11 total, 11 up, 11 in 2023-07-14T11:08:36.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:36 smithi040 ceph-mon[97954]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:36.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:36 smithi040 ceph-mon[97954]: osdmap e74: 11 total, 11 up, 11 in 2023-07-14T11:08:36.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:36 smithi045 ceph-mon[128791]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:08:36.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:36 smithi045 ceph-mon[128791]: osdmap e74: 11 total, 11 up, 11 in 2023-07-14T11:08:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:37 smithi130 ceph-mon[129405]: pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:37 smithi040 ceph-mon[97954]: pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:37 smithi045 ceph-mon[128791]: pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: osdmap e75: 11 total, 11 up, 11 in 2023-07-14T11:08:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/2828275402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]': finished 2023-07-14T11:08:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: osdmap e76: 12 total, 11 up, 12 in 2023-07-14T11:08:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:38 smithi130 ceph-mon[129405]: from='client.? 172.21.15.130:0/3522229223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:38.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: osdmap e75: 11 total, 11 up, 11 in 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/2828275402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]': finished 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: osdmap e76: 12 total, 11 up, 12 in 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:38 smithi040 ceph-mon[97954]: from='client.? 172.21.15.130:0/3522229223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: osdmap e75: 11 total, 11 up, 11 in 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/2828275402' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]: dispatch 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ccf993ad-d29e-44b7-9aa1-bf856fc9a92e"}]': finished 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: osdmap e76: 12 total, 11 up, 12 in 2023-07-14T11:08:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:38 smithi045 ceph-mon[128791]: from='client.? 172.21.15.130:0/3522229223' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-07-14T11:08:39.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:39 smithi130 ceph-mon[129405]: pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:39.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:39 smithi040 ceph-mon[97954]: pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:39.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:39 smithi045 ceph-mon[128791]: pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 299 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:41.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:41 smithi130 ceph-mon[129405]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:08:42.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:41 smithi040 ceph-mon[97954]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:08:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:41 smithi045 ceph-mon[128791]: pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 109 KiB/s, 0 objects/s recovering 2023-07-14T11:08:43.945 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:43 smithi130 ceph-mon[129405]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:08:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:43 smithi040 ceph-mon[97954]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:08:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:43 smithi045 ceph-mon[128791]: pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 92 KiB/s, 0 objects/s recovering 2023-07-14T11:08:44.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-07-14T11:08:44.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:44 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-07-14T11:08:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:44 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:45.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "osd.11"}]: dispatch 2023-07-14T11:08:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:44 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:45.989 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:45 smithi130 ceph-mon[129405]: Deploying daemon osd.11 on smithi130 2023-07-14T11:08:45.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:45 smithi130 ceph-mon[129405]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:08:46.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:45 smithi040 ceph-mon[97954]: Deploying daemon osd.11 on smithi130 2023-07-14T11:08:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:45 smithi040 ceph-mon[97954]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:08:46.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:45 smithi045 ceph-mon[128791]: Deploying daemon osd.11 on smithi130 2023-07-14T11:08:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:45 smithi045 ceph-mon[128791]: pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-07-14T11:08:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:47 smithi130 ceph-mon[129405]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-07-14T11:08:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:47 smithi040 ceph-mon[97954]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-07-14T11:08:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:47 smithi045 ceph-mon[128791]: pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 63 KiB/s, 0 objects/s recovering 2023-07-14T11:08:49.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:49 smithi130 ceph-mon[129405]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-07-14T11:08:49.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:49.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:49.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:49 smithi045 ceph-mon[128791]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-07-14T11:08:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:49 smithi040 ceph-mon[97954]: pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-07-14T11:08:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:50.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:50.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:51.572 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.569+0000 7ff8b7fff700 1 -- 172.21.15.130:0/3531473548 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+38 (secure 0 0 0) 0x7ff8d0060950 con 0x7ff8b80a1150 2023-07-14T11:08:51.572 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 11 on host 'smithi130' 2023-07-14T11:08:51.574 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 msgr2=0x7ff8b80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:51.574 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 0x7ff8b80a3600 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7ff8c000b2e0 tx=0x7ff8c0005e80 comp rx=0 tx=0).stop 2023-07-14T11:08:51.574 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 msgr2=0x7ff8d019cc30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:51.574 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7ff8c8007ee0 tx=0x7ff8c800d3b0 comp rx=0 tx=0).stop 2023-07-14T11:08:51.574 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 shutdown_connections 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8b80a1150 0x7ff8b80a3600 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8d0100610 0x7ff8d019cc30 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.571+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8d00fea80 0x7ff8d0068b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.572+0000 7ff8d51bc700 1 --2- 172.21.15.130:0/3531473548 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8d00ffc80 0x7ff8d019a5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.572+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 >> 172.21.15.130:0/3531473548 conn(0x7ff8d00fa0b0 msgr2=0x7ff8d00fc380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.572+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 shutdown_connections 2023-07-14T11:08:51.575 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:08:51.572+0000 7ff8d51bc700 1 -- 172.21.15.130:0/3531473548 wait complete. 2023-07-14T11:08:51.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:51 smithi130 ceph-mon[129405]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-07-14T11:08:51.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:51.982 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:51.983 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:51.983 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:51 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:51.984 DEBUG:teuthology.orchestra.run.smithi130:osd.11> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.11.service 2023-07-14T11:08:51.986 INFO:tasks.cephadm:Waiting for 12 OSDs to come up... 2023-07-14T11:08:51.987 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd stat -f json 2023-07-14T11:08:52.026 INFO:journalctl@ceph.osd.11.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:08:52.058 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:51 smithi040 ceph-mon[97954]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-07-14T11:08:52.058 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.058 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.058 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.059 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:51 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:51 smithi045 ceph-mon[128791]: pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-07-14T11:08:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:51 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:52.729 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:08:53.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:52 smithi130 ceph-mon[129405]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:52 smithi130 ceph-mon[129405]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:53.043 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:08:52 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:08:52.730+0000 7f7e60b32640 -1 osd.11 0 log_to_monitors true 2023-07-14T11:08:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:52 smithi045 ceph-mon[128791]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:52 smithi045 ceph-mon[128791]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:53.099 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:52 smithi040 ceph-mon[97954]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:53.099 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:52 smithi040 ceph-mon[97954]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]: dispatch 2023-07-14T11:08:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-07-14T11:08:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: osdmap e77: 12 total, 11 up, 12 in 2023-07-14T11:08:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:53 smithi045 ceph-mon[128791]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.217 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:08:53.813+0000 7f7e561aa700 -1 osd.11 0 waiting for initial osdmap 2023-07-14T11:08:54.218 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:08:53.826+0000 7f7e51f9f700 -1 osd.11 78 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-07-14T11:08:54.218 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:54.218 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-07-14T11:08:54.218 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: osdmap e77: 12 total, 11 up, 12 in 2023-07-14T11:08:54.218 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:54.219 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.219 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:53 smithi130 ceph-mon[129405]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.218+0000 7f3a346ea700 1 -- 172.21.15.40:0/2930237047 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c072f70 msgr2=0x7f3a2c0733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.218+0000 7f3a346ea700 1 --2- 172.21.15.40:0/2930237047 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c072f70 0x7f3a2c0733e0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f3a1c009990 tx=0x7f3a1c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 -- 172.21.15.40:0/2930237047 shutdown_connections 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 --2- 172.21.15.40:0/2930237047 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c107a60 0x7f3a2c109ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 --2- 172.21.15.40:0/2930237047 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c074540 0x7f3a2c0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 --2- 172.21.15.40:0/2930237047 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c072f70 0x7f3a2c0733e0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 -- 172.21.15.40:0/2930237047 >> 172.21.15.40:0/2930237047 conn(0x7f3a2c0fb1a0 msgr2=0x7f3a2c0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 -- 172.21.15.40:0/2930237047 shutdown_connections 2023-07-14T11:08:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.219+0000 7f3a346ea700 1 -- 172.21.15.40:0/2930237047 wait complete. 2023-07-14T11:08:54.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 Processor -- start 2023-07-14T11:08:54.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 -- start start 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 0x7f3a2c197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 0x7f3a2c19f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c19f660 con 0x7f3a2c107a60 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c19f7a0 con 0x7f3a2c072f70 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.220+0000 7f3a346ea700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c19f930 con 0x7f3a2c074540 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a32c87700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 0x7f3a2c19f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a32486700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 0x7f3a2c197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57752/0 (socket says 172.21.15.40:57752) 2023-07-14T11:08:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a32486700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 0x7f3a2c197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:51262/0 (socket says 172.21.15.40:51262) 2023-07-14T11:08:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 -- 172.21.15.40:0/3687273383 learned_addr learned my addr 172.21.15.40:0/3687273383 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:08:54.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 -- 172.21.15.40:0/3687273383 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 msgr2=0x7f3a2c197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:54.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 0x7f3a2c197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 -- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 msgr2=0x7f3a2c19f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:54.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 0x7f3a2c19f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 -- 172.21.15.40:0/3687273383 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a1c0095f0 con 0x7f3a2c074540 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a32c87700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 0x7f3a2c19f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.221+0000 7f3a31c85700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f3a1c004770 tx=0x7f3a1c009580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.222+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a1c01d070 con 0x7f3a2c074540 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.222+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3a1c005030 con 0x7f3a2c074540 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.222+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a2c19fbb0 con 0x7f3a2c074540 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.222+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a1c004b50 con 0x7f3a2c074540 2023-07-14T11:08:54.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.222+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a2c19ffc0 con 0x7f3a2c074540 2023-07-14T11:08:54.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.223+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a2c106110 con 0x7f3a2c074540 2023-07-14T11:08:54.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.225+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f3a1c004cf0 con 0x7f3a2c074540 2023-07-14T11:08:54.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.226+0000 7f3a237fe700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 0x7f3a180a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:54.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.226+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(78..78 src has 1..78) v4 ==== 6847+0+0 (secure 0 0 0) 0x7f3a1c0cd480 con 0x7f3a2c074540 2023-07-14T11:08:54.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.226+0000 7f3a32486700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 0x7f3a180a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:54.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.227+0000 7f3a32486700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 0x7f3a180a3680 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009c60 tx=0x7f3a28009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:54.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.228+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3a1c097180 con 0x7f3a2c074540 2023-07-14T11:08:54.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["11"]}]': finished 2023-07-14T11:08:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: osdmap e77: 12 total, 11 up, 12 in 2023-07-14T11:08:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: from='osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935]' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:53 smithi040 ceph-mon[97954]: from='osd.11 ' entity='osd.11' cmd=[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2023-07-14T11:08:54.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.455+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7f3a2c02caa0 con 0x7f3a2c074540 2023-07-14T11:08:54.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.456+0000 7f3a237fe700 1 -- 172.21.15.40:0/3687273383 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v78) v1 ==== 74+0+133 (secure 0 0 0) 0x7f3a1c09a7a0 con 0x7f3a2c074540 2023-07-14T11:08:54.458 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 msgr2=0x7f3a180a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 0x7f3a180a3680 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f3a28009c60 tx=0x7f3a28009260 comp rx=0 tx=0).stop 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 msgr2=0x7f3a2c198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f3a1c004770 tx=0x7f3a1c009580 comp rx=0 tx=0).stop 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 shutdown_connections 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a180a11d0 0x7f3a180a3680 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c107a60 0x7f3a2c19f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c072f70 0x7f3a2c197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 --2- 172.21.15.40:0/3687273383 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c074540 0x7f3a2c198010 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:54.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 >> 172.21.15.40:0/3687273383 conn(0x7f3a2c0fb1a0 msgr2=0x7f3a2c0fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:54.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 shutdown_connections 2023-07-14T11:08:54.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:54.459+0000 7f3a346ea700 1 -- 172.21.15.40:0/3687273383 wait complete. 2023-07-14T11:08:55.010 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":78,"num_osds":12,"num_up_osds":11,"osd_up_since":1689332914,"num_in_osds":12,"osd_in_since":1689332917,"num_remapped_pgs":0} 2023-07-14T11:08:55.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:08:55.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: osdmap e78: 12 total, 11 up, 12 in 2023-07-14T11:08:55.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3687273383' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:54 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: osdmap e78: 12 total, 11 up, 12 in 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3687273383' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:54 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='osd.11 ' entity='osd.11' cmd='[{"prefix": "osd crush create-or-move", "id": 11, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2023-07-14T11:08:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: osdmap e78: 12 total, 11 up, 12 in 2023-07-14T11:08:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:55.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3687273383' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:08:55.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:54 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:08:56.011 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd stat -f json 2023-07-14T11:08:56.082 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: purged_snaps scrub starts 2023-07-14T11:08:56.082 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: purged_snaps scrub ok 2023-07-14T11:08:56.082 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: Detected new or changed devices on smithi130 2023-07-14T11:08:56.083 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 4292M 2023-07-14T11:08:56.083 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:56.083 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] boot 2023-07-14T11:08:56.083 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: osdmap e79: 12 total, 12 up, 12 in 2023-07-14T11:08:56.083 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:55 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:56.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: purged_snaps scrub starts 2023-07-14T11:08:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: purged_snaps scrub ok 2023-07-14T11:08:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: Detected new or changed devices on smithi130 2023-07-14T11:08:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 4292M 2023-07-14T11:08:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:56.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] boot 2023-07-14T11:08:56.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: osdmap e79: 12 total, 12 up, 12 in 2023-07-14T11:08:56.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:55 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:56.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: purged_snaps scrub starts 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: purged_snaps scrub ok 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: Detected new or changed devices on smithi130 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 4292M 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 300 MiB used, 983 GiB / 983 GiB avail 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: osd.11 [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] boot 2023-07-14T11:08:56.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: osdmap e79: 12 total, 12 up, 12 in 2023-07-14T11:08:56.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:55 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:08:56.757 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:08:57.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:57 smithi040 ceph-mon[97954]: osdmap e80: 12 total, 12 up, 12 in 2023-07-14T11:08:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:57 smithi040 ceph-mon[97954]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:08:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:57 smithi045 ceph-mon[128791]: osdmap e80: 12 total, 12 up, 12 in 2023-07-14T11:08:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:57 smithi045 ceph-mon[128791]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:08:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:57 smithi130 ceph-mon[129405]: osdmap e80: 12 total, 12 up, 12 in 2023-07-14T11:08:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:57 smithi130 ceph-mon[129405]: pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:08:58.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.252+0000 7fc00e876700 1 -- 172.21.15.40:0/1093905305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008100d90 msgr2=0x7fc0081011e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:58.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.252+0000 7fc00e876700 1 --2- 172.21.15.40:0/1093905305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008100d90 0x7fc0081011e0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fbffc0098f0 tx=0x7fbffc009c00 comp rx=0 tx=0).stop 2023-07-14T11:08:58.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.252+0000 7fc00e876700 1 -- 172.21.15.40:0/1093905305 shutdown_connections 2023-07-14T11:08:58.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 --2- 172.21.15.40:0/1093905305 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc008101720 0x7fc008107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 --2- 172.21.15.40:0/1093905305 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc0080ffb90 0x7fc0080fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 --2- 172.21.15.40:0/1093905305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008100d90 0x7fc0081011e0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 -- 172.21.15.40:0/1093905305 >> 172.21.15.40:0/1093905305 conn(0x7fc0080fb1a0 msgr2=0x7fc0080fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:58.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 -- 172.21.15.40:0/1093905305 shutdown_connections 2023-07-14T11:08:58.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.253+0000 7fc00e876700 1 -- 172.21.15.40:0/1093905305 wait complete. 2023-07-14T11:08:58.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 Processor -- start 2023-07-14T11:08:58.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 -- start start 2023-07-14T11:08:58.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc008100d90 0x7fc008072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008101720 0x7fc008072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0080735c0 con 0x7fc008101720 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc008073730 con 0x7fc008100d90 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00e876700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc00819f990 con 0x7fc0080ffb90 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc007fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc00ce13700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008101720 0x7fc008072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc007fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57762/0 (socket says 172.21.15.40:57762) 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.254+0000 7fc007fff700 1 -- 172.21.15.40:0/3308115348 learned_addr learned my addr 172.21.15.40:0/3308115348 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc0077fe700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc008100d90 0x7fc008072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 -- 172.21.15.40:0/3308115348 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc008100d90 msgr2=0x7fc008072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc008100d90 0x7fc008072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 -- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008101720 msgr2=0x7fc008072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008101720 0x7fc008072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 -- 172.21.15.40:0/3308115348 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbffc009520 con 0x7fc0080ffb90 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc007fff700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fbff800e970 tx=0x7fbff800ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff8009790 con 0x7fc0080ffb90 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbff800c7e0 con 0x7fc0080ffb90 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.255+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc00819fc70 con 0x7fc0080ffb90 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.256+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbff8003c70 con 0x7fc0080ffb90 2023-07-14T11:08:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.256+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0081a0170 con 0x7fc0080ffb90 2023-07-14T11:08:58.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.257+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc00810b3a0 con 0x7fc0080ffb90 2023-07-14T11:08:58.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.259+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fbff800cc50 con 0x7fc0080ffb90 2023-07-14T11:08:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.260+0000 7fc0057fa700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 0x7fbff00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:08:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.260+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fbff8014070 con 0x7fc0080ffb90 2023-07-14T11:08:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.260+0000 7fc0077fe700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 0x7fbff00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:08:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.261+0000 7fc0077fe700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 0x7fbff00a3600 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fbffc005e90 tx=0x7fbffc019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:08:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.262+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fbff8096350 con 0x7fc0080ffb90 2023-07-14T11:08:58.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.468+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7fc00810b590 con 0x7fc0080ffb90 2023-07-14T11:08:58.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.469+0000 7fc0057fa700 1 -- 172.21.15.40:0/3308115348 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v80) v1 ==== 74+0+133 (secure 0 0 0) 0x7fbff8003dd0 con 0x7fc0080ffb90 2023-07-14T11:08:58.470 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:08:58.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 msgr2=0x7fbff00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:58.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 0x7fbff00a3600 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fbffc005e90 tx=0x7fbffc019040 comp rx=0 tx=0).stop 2023-07-14T11:08:58.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 msgr2=0x7fc0080742b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:08:58.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7fbff800e970 tx=0x7fbff800ec80 comp rx=0 tx=0).stop 2023-07-14T11:08:58.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 shutdown_connections 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbff00a1150 0x7fbff00a3600 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc008101720 0x7fc008072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0080ffb90 0x7fc0080742b0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 --2- 172.21.15.40:0/3308115348 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc008100d90 0x7fc008072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 >> 172.21.15.40:0/3308115348 conn(0x7fc0080fb1a0 msgr2=0x7fc0080fd480 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 shutdown_connections 2023-07-14T11:08:58.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:08:58.471+0000 7fc00e876700 1 -- 172.21.15.40:0/3308115348 wait complete. 2023-07-14T11:08:58.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3308115348' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3308115348' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:58.935 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":80,"num_osds":12,"num_up_osds":12,"osd_up_since":1689332934,"num_in_osds":12,"osd_in_since":1689332917,"num_remapped_pgs":0} 2023-07-14T11:08:58.936 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd dump --format=json 2023-07-14T11:08:59.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3308115348' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-07-14T11:08:59.669 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:08:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:08:59 smithi045 ceph-mon[128791]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:08:59.920 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:08:59 smithi040 ceph-mon[97954]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:08:59 smithi130 ceph-mon[129405]: pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:01.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 -- 172.21.15.40:0/268651707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 msgr2=0x7f3a2c10ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:01.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 --2- 172.21.15.40:0/268651707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c10ca20 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f3a2400b0d0 tx=0x7f3a2400b3e0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 -- 172.21.15.40:0/268651707 shutdown_connections 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 --2- 172.21.15.40:0/268651707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c10ca20 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 --2- 172.21.15.40:0/268651707 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c1001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 --2- 172.21.15.40:0/268651707 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 -- 172.21.15.40:0/268651707 >> 172.21.15.40:0/268651707 conn(0x7f3a2c0fb660 msgr2=0x7f3a2c0fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 -- 172.21.15.40:0/268651707 shutdown_connections 2023-07-14T11:09:01.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.053+0000 7f3a325db700 1 -- 172.21.15.40:0/268651707 wait complete. 2023-07-14T11:09:01.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.054+0000 7f3a325db700 1 Processor -- start 2023-07-14T11:09:01.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.054+0000 7f3a325db700 1 -- start start 2023-07-14T11:09:01.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.054+0000 7f3a325db700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:01.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a325db700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a325db700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a325db700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c1a41d0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a325db700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c1a4340 con 0x7f3a2c1007a0 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a325db700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a2c1a4500 con 0x7f3a2c0ffe00 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56228/0 (socket says 172.21.15.40:56228) 2023-07-14T11:09:01.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 -- 172.21.15.40:0/216074087 learned_addr learned my addr 172.21.15.40:0/216074087 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a2b7fe700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a2bfff700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 -- 172.21.15.40:0/216074087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 msgr2=0x7f3a2c072810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c072810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 -- 172.21.15.40:0/216074087 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 msgr2=0x7f3a2c072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a30b78700 1 -- 172.21.15.40:0/216074087 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a24009d00 con 0x7f3a2c104ea0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a2bfff700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c072810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.055+0000 7f3a2b7fe700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a30b78700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f3a24007890 tx=0x7f3a240078c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a2400e040 con 0x7f3a2c104ea0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a2c1a4780 con 0x7f3a2c104ea0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3a2400bdb0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3a2401d470 con 0x7f3a2c104ea0 2023-07-14T11:09:01.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.056+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a2c1a4c40 con 0x7f3a2c104ea0 2023-07-14T11:09:01.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.057+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a2c04e210 con 0x7f3a2c104ea0 2023-07-14T11:09:01.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.059+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f3a240050b0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.061+0000 7f3a297fa700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 0x7f3a140a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:01.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.061+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f3a240cd9f0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.061+0000 7f3a2bfff700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 0x7f3a140a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:01.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.061+0000 7f3a2bfff700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 0x7f3a140a3600 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f3a1c009cb0 tx=0x7f3a1c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:01.064 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.063+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3a240975d0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.272+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f3a2c1a4e60 con 0x7f3a2c104ea0 2023-07-14T11:09:01.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.273+0000 7f3a297fa700 1 -- 172.21.15.40:0/216074087 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) v1 ==== 74+0+18361 (secure 0 0 0) 0x7f3a2409abf0 con 0x7f3a2c104ea0 2023-07-14T11:09:01.275 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:01.275 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":80,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:08:56.533044+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:01.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.275+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 msgr2=0x7f3a140a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:01.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.275+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 0x7f3a140a3600 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f3a1c009cb0 tx=0x7f3a1c009260 comp rx=0 tx=0).stop 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.275+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 msgr2=0x7f3a2c0732d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.275+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f3a24007890 tx=0x7f3a240078c0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 shutdown_connections 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3a140a1150 0x7f3a140a3600 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3a2c104ea0 0x7f3a2c0732d0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3a2c0ffe00 0x7f3a2c072810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 --2- 172.21.15.40:0/216074087 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3a2c1007a0 0x7f3a2c072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:01.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 >> 172.21.15.40:0/216074087 conn(0x7f3a2c0fb660 msgr2=0x7f3a2c0fc080 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:01.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 shutdown_connections 2023-07-14T11:09:01.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:01.276+0000 7f3a325db700 1 -- 172.21.15.40:0/216074087 wait complete. 2023-07-14T11:09:01.748 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-07-14T11:09:01.749 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd pool get .mgr pg_num 2023-07-14T11:09:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:01 smithi130 ceph-mon[129405]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:01 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/216074087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:01 smithi040 ceph-mon[97954]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:01 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/216074087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:01 smithi045 ceph-mon[128791]: pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:01 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/216074087' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:02.480 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:09:03.932 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:03 smithi040 ceph-mon[97954]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:03 smithi130 ceph-mon[129405]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:03 smithi045 ceph-mon[128791]: pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:04.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/852221435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8101550 msgr2=0x7ff7f8109c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/852221435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8101550 0x7ff7f8109c20 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7ff7e8009990 tx=0x7ff7e8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/852221435 shutdown_connections 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/852221435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f810a160 0x7ff7f810c5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/852221435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f8100c40 0x7ff7f8101010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/852221435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8101550 0x7ff7f8109c20 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/852221435 >> 172.21.15.40:0/852221435 conn(0x7ff7f8075b70 msgr2=0x7ff7f8075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:04.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/852221435 shutdown_connections 2023-07-14T11:09:04.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.154+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/852221435 wait complete. 2023-07-14T11:09:04.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.155+0000 7ff7ff8b6700 1 Processor -- start 2023-07-14T11:09:04.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.155+0000 7ff7ff8b6700 1 -- start start 2023-07-14T11:09:04.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:04.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 0x7ff7f80fefb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:04.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 0x7ff7f80ff530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7f81a40f0 con 0x7ff7f8101550 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7f81a4260 con 0x7ff7f810a160 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7ff8b6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7f81a4420 con 0x7ff7f8100c40 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7fd652700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7fce51700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 0x7ff7f80fefb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7fd652700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:38614/0 (socket says 172.21.15.40:38614) 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.156+0000 7ff7fd652700 1 -- 172.21.15.40:0/374178993 learned_addr learned my addr 172.21.15.40:0/374178993 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:04.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fde53700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 0x7ff7f80ff530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:04.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 -- 172.21.15.40:0/374178993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 msgr2=0x7ff7f80ff530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 0x7ff7f80ff530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 -- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 msgr2=0x7ff7f80fefb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 0x7ff7f80fefb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 -- 172.21.15.40:0/374178993 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff7f4009520 con 0x7ff7f8100c40 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fde53700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 0x7ff7f80ff530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fce51700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 0x7ff7f80fefb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7fd652700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7ff7f400eb30 tx=0x7ff7f400ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:04.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.157+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff7f4009880 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.158+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff7f0009c40 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.158+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff7f40044e0 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.158+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff7f4003e90 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.158+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff7e80095f0 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.159+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff7f804e210 con 0x7ff7f8100c40 2023-07-14T11:09:04.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.161+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7ff7f4004000 con 0x7ff7f8100c40 2023-07-14T11:09:04.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.162+0000 7ff7ee7fc700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 0x7ff7e40a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:04.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.163+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7ff7f4014070 con 0x7ff7f8100c40 2023-07-14T11:09:04.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.163+0000 7ff7fce51700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 0x7ff7e40a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:04.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.163+0000 7ff7fce51700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 0x7ff7e40a3600 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7ff7e800b2e0 tx=0x7ff7e8005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:04.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.164+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff7f4096620 con 0x7ff7f8100c40 2023-07-14T11:09:04.371 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.369+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7ff7f81a4b30 con 0x7ff7f8100c40 2023-07-14T11:09:04.371 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.370+0000 7ff7ee7fc700 1 -- 172.21.15.40:0/374178993 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v80) v1 ==== 93+0+10 (secure 0 0 0) 0x7ff7f400f3c0 con 0x7ff7f8100c40 2023-07-14T11:09:04.372 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:09:04.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 msgr2=0x7ff7e40a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:04.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 0x7ff7e40a3600 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7ff7e800b2e0 tx=0x7ff7e8005960 comp rx=0 tx=0).stop 2023-07-14T11:09:04.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 msgr2=0x7ff7f80fea70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:04.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7ff7f400eb30 tx=0x7ff7f400ee40 comp rx=0 tx=0).stop 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 shutdown_connections 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff7e40a1150 0x7ff7e40a3600 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff7f810a160 0x7ff7f80ff530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff7f8100c40 0x7ff7f80fea70 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 --2- 172.21.15.40:0/374178993 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff7f8101550 0x7ff7f80fefb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 >> 172.21.15.40:0/374178993 conn(0x7ff7f8075b70 msgr2=0x7ff7f80fd340 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 shutdown_connections 2023-07-14T11:09:04.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:04.372+0000 7ff7ff8b6700 1 -- 172.21.15.40:0/374178993 wait complete. 2023-07-14T11:09:04.808 INFO:tasks.cephadm:Adding mds.a on smithi040 2023-07-14T11:09:04.809 INFO:tasks.cephadm:Adding mds.d on smithi040 2023-07-14T11:09:04.809 INFO:tasks.cephadm:Adding mds.g on smithi040 2023-07-14T11:09:04.809 INFO:tasks.cephadm:Adding mds.j on smithi040 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.b on smithi045 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.e on smithi045 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.h on smithi045 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.k on smithi045 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.c on smithi130 2023-07-14T11:09:04.810 INFO:tasks.cephadm:Adding mds.f on smithi130 2023-07-14T11:09:04.811 INFO:tasks.cephadm:Adding mds.i on smithi130 2023-07-14T11:09:04.811 INFO:tasks.cephadm:Adding mds.l on smithi130 2023-07-14T11:09:04.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph orch apply mds all '12;smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l' 2023-07-14T11:09:05.378 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:09:05.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:05 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/374178993' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:05.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:05 smithi130 ceph-mon[129405]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:05.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:05 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/374178993' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:05.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:05 smithi040 ceph-mon[97954]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:05 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/374178993' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:05 smithi045 ceph-mon[128791]: pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:06.590 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 -- 172.21.15.130:0/237544546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa538100610 msgr2=0x7fa538106ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:06.590 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/237544546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa538100610 0x7fa538106ed0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fa53000b0a0 tx=0x7fa53000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 -- 172.21.15.130:0/237544546 shutdown_connections 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/237544546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa538100610 0x7fa538106ed0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/237544546 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5380fea80 0x7fa5380fee90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/237544546 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380ffc80 0x7fa5381000d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 -- 172.21.15.130:0/237544546 >> 172.21.15.130:0/237544546 conn(0x7fa5380fa0b0 msgr2=0x7fa5380fc500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 -- 172.21.15.130:0/237544546 shutdown_connections 2023-07-14T11:09:06.591 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 -- 172.21.15.130:0/237544546 wait complete. 2023-07-14T11:09:06.593 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.588+0000 7fa53fb4d700 1 Processor -- start 2023-07-14T11:09:06.593 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- start start 2023-07-14T11:09:06.593 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380fea80 0x7fa538068b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:06.593 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 0x7fa53819cda0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa53819d540 con 0x7fa538100610 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa53819d6b0 con 0x7fa5380fea80 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa53819d870 con 0x7fa5380ffc80 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.130:39804/0 (socket says 172.21.15.130:39804) 2023-07-14T11:09:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 -- 172.21.15.130:0/3454815585 learned_addr learned my addr 172.21.15.130:0/3454815585 (peer_addr_for_me v2:172.21.15.130:0/0) 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d8e9700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380fea80 0x7fa538068b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53e0ea700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 0x7fa53819cda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 -- 172.21.15.130:0/3454815585 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380fea80 msgr2=0x7fa538068b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380fea80 0x7fa538068b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 -- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 msgr2=0x7fa53819cda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 0x7fa53819cda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 -- 172.21.15.130:0/3454815585 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa530009d00 con 0x7fa5380ffc80 2023-07-14T11:09:06.595 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53e0ea700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 0x7fa53819cda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53d0e8700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fa53400d950 tx=0x7fa53400dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa534009940 con 0x7fa5380ffc80 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa534010460 con 0x7fa5380ffc80 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa53819db50 con 0x7fa5380ffc80 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53400f620 con 0x7fa5380ffc80 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.589+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa53819e0a0 con 0x7fa5380ffc80 2023-07-14T11:09:06.596 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.590+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa538105fc0 con 0x7fa5380ffc80 2023-07-14T11:09:06.597 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.591+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa5340108d0 con 0x7fa5380ffc80 2023-07-14T11:09:06.597 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.592+0000 7fa52effd700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 0x7fa5240a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:06.597 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.592+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fa5340cc220 con 0x7fa5380ffc80 2023-07-14T11:09:06.598 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.592+0000 7fa53d8e9700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 0x7fa5240a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:06.598 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.593+0000 7fa53d8e9700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 0x7fa5240a3680 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fa5380ffae0 tx=0x7fa528009190 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:06.598 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.595+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa534095e10 con 0x7fa5380ffc80 2023-07-14T11:09:06.798 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.796+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}) v1 -- 0x7fa538060c10 con 0x7fa5240a11d0 2023-07-14T11:09:06.808 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.806+0000 7fa52effd700 1 -- 172.21.15.130:0/3454815585 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+28 (secure 0 0 0) 0x7fa538060c10 con 0x7fa5240a11d0 2023-07-14T11:09:06.809 INFO:teuthology.orchestra.run.smithi130.stdout:Scheduled mds.all update... 2023-07-14T11:09:06.811 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 msgr2=0x7fa5240a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:06.811 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 0x7fa5240a3680 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fa5380ffae0 tx=0x7fa528009190 comp rx=0 tx=0).stop 2023-07-14T11:09:06.811 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 msgr2=0x7fa53819a750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:06.811 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fa53400d950 tx=0x7fa53400dc60 comp rx=0 tx=0).stop 2023-07-14T11:09:06.811 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 shutdown_connections 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.808+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5240a11d0 0x7fa5240a3680 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa538100610 0x7fa53819cda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa5380fea80 0x7fa538068b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 --2- 172.21.15.130:0/3454815585 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa5380ffc80 0x7fa53819a750 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 >> 172.21.15.130:0/3454815585 conn(0x7fa5380fa0b0 msgr2=0x7fa538103100 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 shutdown_connections 2023-07-14T11:09:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:2023-07-14T11:09:06.809+0000 7fa53fb4d700 1 -- 172.21.15.130:0/3454815585 wait complete. 2023-07-14T11:09:07.232 DEBUG:teuthology.orchestra.run.smithi040:mds.a> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.a.service 2023-07-14T11:09:07.237 DEBUG:teuthology.orchestra.run.smithi040:mds.d> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.d.service 2023-07-14T11:09:07.282 DEBUG:teuthology.orchestra.run.smithi040:mds.g> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.g.service 2023-07-14T11:09:07.322 INFO:journalctl@ceph.mds.a.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:09:07.325 DEBUG:teuthology.orchestra.run.smithi040:mds.j> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.j.service 2023-07-14T11:09:07.359 INFO:journalctl@ceph.mds.d.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:09:07.362 DEBUG:teuthology.orchestra.run.smithi045:mds.b> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.b.service 2023-07-14T11:09:07.364 DEBUG:teuthology.orchestra.run.smithi045:mds.e> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.e.service 2023-07-14T11:09:07.394 INFO:journalctl@ceph.mds.b.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:09:07.396 DEBUG:teuthology.orchestra.run.smithi045:mds.h> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.h.service 2023-07-14T11:09:07.408 INFO:journalctl@ceph.mds.g.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:09:07.422 INFO:journalctl@ceph.mds.e.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:09:07.423 DEBUG:teuthology.orchestra.run.smithi045:mds.k> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.k.service 2023-07-14T11:09:07.432 INFO:journalctl@ceph.mds.j.smithi040.stdout:-- Logs begin at Fri 2023-07-14 10:41:25 UTC. -- 2023-07-14T11:09:07.449 INFO:journalctl@ceph.mds.h.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:09:07.450 DEBUG:teuthology.orchestra.run.smithi130:mds.c> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.c.service 2023-07-14T11:09:07.452 DEBUG:teuthology.orchestra.run.smithi130:mds.f> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.f.service 2023-07-14T11:09:07.477 INFO:journalctl@ceph.mds.k.smithi045.stdout:-- Logs begin at Fri 2023-07-14 10:36:44 UTC. -- 2023-07-14T11:09:07.482 INFO:journalctl@ceph.mds.c.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:09:07.483 DEBUG:teuthology.orchestra.run.smithi130:mds.i> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.i.service 2023-07-14T11:09:07.508 INFO:journalctl@ceph.mds.f.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:09:07.509 DEBUG:teuthology.orchestra.run.smithi130:mds.l> sudo journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.l.service 2023-07-14T11:09:07.534 INFO:journalctl@ceph.mds.i.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:09:07.535 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2023-07-14T11:09:07.536 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:07.560 INFO:journalctl@ceph.mds.l.smithi130.stdout:-- Logs begin at Fri 2023-07-14 10:32:05 UTC. -- 2023-07-14T11:09:07.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.753+0000 7f78bbfff700 1 -- 172.21.15.40:0/3381451429 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b4072f70 msgr2=0x7f78b40733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.753+0000 7f78bbfff700 1 --2- 172.21.15.40:0/3381451429 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b4072f70 0x7f78b40733e0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f78a4009990 tx=0x7f78a4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 -- 172.21.15.40:0/3381451429 shutdown_connections 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 --2- 172.21.15.40:0/3381451429 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b41079c0 0x7f78b4109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 --2- 172.21.15.40:0/3381451429 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 0x7f78b40729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 --2- 172.21.15.40:0/3381451429 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b4072f70 0x7f78b40733e0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 -- 172.21.15.40:0/3381451429 >> 172.21.15.40:0/3381451429 conn(0x7f78b40fb0e0 msgr2=0x7f78b40fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:07.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 -- 172.21.15.40:0/3381451429 shutdown_connections 2023-07-14T11:09:07.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.754+0000 7f78bbfff700 1 -- 172.21.15.40:0/3381451429 wait complete. 2023-07-14T11:09:07.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.755+0000 7f78bbfff700 1 Processor -- start 2023-07-14T11:09:07.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.755+0000 7f78bbfff700 1 -- start start 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 0x7f78b4197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 0x7f78b4197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78b419f5e0 con 0x7f78b41079c0 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78b419f720 con 0x7f78b4072f70 2023-07-14T11:09:07.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78bbfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78b419f8b0 con 0x7f78b4074550 2023-07-14T11:09:07.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78ba59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:07.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78ba59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54704/0 (socket says 172.21.15.40:54704) 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78ba59c700 1 -- 172.21.15.40:0/4215013464 learned_addr learned my addr 172.21.15.40:0/4215013464 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78b9d9b700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 0x7f78b4197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.756+0000 7f78b959a700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 0x7f78b4197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 -- 172.21.15.40:0/4215013464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 msgr2=0x7f78b4197fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 0x7f78b4197fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 -- 172.21.15.40:0/4215013464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 msgr2=0x7f78b4197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:07.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 0x7f78b4197a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 -- 172.21.15.40:0/4215013464 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78a40095f0 con 0x7f78b41079c0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78b9d9b700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 0x7f78b4197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78ba59c700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f78ac00ef30 tx=0x7f78ac00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78ac018070 con 0x7f78b41079c0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f78ac00f040 con 0x7f78b41079c0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f78ac00cd00 con 0x7f78b41079c0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78b419fb30 con 0x7f78b41079c0 2023-07-14T11:09:07.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.757+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78b41a0040 con 0x7f78b41079c0 2023-07-14T11:09:07.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.758+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78b4106070 con 0x7f78b41079c0 2023-07-14T11:09:07.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.760+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f78ac003a00 con 0x7f78b41079c0 2023-07-14T11:09:07.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.762+0000 7f78aaffd700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 0x7f78a00a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:07.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.762+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f78ac0cc910 con 0x7f78b41079c0 2023-07-14T11:09:07.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.762+0000 7f78b9d9b700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 0x7f78a00a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:07.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.763+0000 7f78b9d9b700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 0x7f78a00a36d0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f78b4074920 tx=0x7f78b0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:07.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.764+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f78ac0964f0 con 0x7f78b41079c0 2023-07-14T11:09:07.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.978+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f78b402c850 con 0x7f78b41079c0 2023-07-14T11:09:07.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.979+0000 7f78aaffd700 1 -- 172.21.15.40:0/4215013464 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) v1 ==== 74+0+18361 (secure 0 0 0) 0x7f78ac099b10 con 0x7f78b41079c0 2023-07-14T11:09:07.982 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:07.982 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":80,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:08:56.533044+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 msgr2=0x7f78a00a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 0x7f78a00a36d0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f78b4074920 tx=0x7f78b0009260 comp rx=0 tx=0).stop 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 msgr2=0x7f78b419f010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f78ac00ef30 tx=0x7f78ac00c3a0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 shutdown_connections 2023-07-14T11:09:07.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78a00a1220 0x7f78a00a36d0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78b41079c0 0x7f78b419f010 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f78b4072f70 0x7f78b4197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 --2- 172.21.15.40:0/4215013464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f78b4074550 0x7f78b4197fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 >> 172.21.15.40:0/4215013464 conn(0x7f78b40fb0e0 msgr2=0x7f78b4109d80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 shutdown_connections 2023-07-14T11:09:07.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:07.982+0000 7f78bbfff700 1 -- 172.21.15.40:0/4215013464 wait complete. 2023-07-14T11:09:08.001 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:08.001 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='client.24442 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: Saving service mds.all spec with placement smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:08.002 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:08.003 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.004 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-07-14T11:09:08.004 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:09:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='client.24442 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:09:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: Saving service mds.all spec with placement smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2023-07-14T11:09:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:08.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='client.24442 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "12;smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: Saving service mds.all spec with placement smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:08.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.l", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:08.115 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 -- 172.21.15.40:0/3921310680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 msgr2=0x7f10e0107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 --2- 172.21.15.40:0/3921310680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e0107f20 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f10d800b0a0 tx=0x7f10d800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 -- 172.21.15.40:0/3921310680 shutdown_connections 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 --2- 172.21.15.40:0/3921310680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e0107f20 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 --2- 172.21.15.40:0/3921310680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e00ffae0 0x7f10e00ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 --2- 172.21.15.40:0/3921310680 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e0100cd0 0x7f10e0101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 -- 172.21.15.40:0/3921310680 >> 172.21.15.40:0/3921310680 conn(0x7f10e00fb0e0 msgr2=0x7f10e00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:08.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 -- 172.21.15.40:0/3921310680 shutdown_connections 2023-07-14T11:09:08.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.209+0000 7f10e83ff700 1 -- 172.21.15.40:0/3921310680 wait complete. 2023-07-14T11:09:08.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.210+0000 7f10e83ff700 1 Processor -- start 2023-07-14T11:09:08.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.210+0000 7f10e83ff700 1 -- start start 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e00ffae0 0x7f10e0197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e0100cd0 0x7f10e01980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10e019f6e0 con 0x7f10e0101660 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10e019f820 con 0x7f10e00ffae0 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e83ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10e019f9b0 con 0x7f10e0100cd0 2023-07-14T11:09:08.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e619b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e00ffae0 0x7f10e0197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54716/0 (socket says 172.21.15.40:54716) 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 -- 172.21.15.40:0/4168619644 learned_addr learned my addr 172.21.15.40:0/4168619644 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e599a700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e0100cd0 0x7f10e01980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 -- 172.21.15.40:0/4168619644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e0100cd0 msgr2=0x7f10e01980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e0100cd0 0x7f10e01980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 -- 172.21.15.40:0/4168619644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e00ffae0 msgr2=0x7f10e0197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e00ffae0 0x7f10e0197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 -- 172.21.15.40:0/4168619644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10d8009d00 con 0x7f10e0101660 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.211+0000 7f10e699c700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f10d800d010 tx=0x7f10d800bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.212+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10d800e040 con 0x7f10e0101660 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.212+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10e019fc30 con 0x7f10e0101660 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.212+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f10d8003e10 con 0x7f10e0101660 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.212+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10d80052a0 con 0x7f10e0101660 2023-07-14T11:09:08.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.212+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10e01a0000 con 0x7f10e0101660 2023-07-14T11:09:08.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.213+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10e010b380 con 0x7f10e0101660 2023-07-14T11:09:08.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.215+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f10d801d480 con 0x7f10e0101660 2023-07-14T11:09:08.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.216+0000 7f10d77fe700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 0x7f10cc0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.216+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f10d80cd860 con 0x7f10e0101660 2023-07-14T11:09:08.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.216+0000 7f10e619b700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 0x7f10cc0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.217+0000 7f10e619b700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 0x7f10cc0a3600 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f10e0100b30 tx=0x7f10dc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:08.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.220+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f10d8097440 con 0x7f10e0101660 2023-07-14T11:09:08.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.427+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f10e01a05b0 con 0x7f10e0101660 2023-07-14T11:09:08.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.428+0000 7f10d77fe700 1 -- 172.21.15.40:0/4168619644 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v80) v1 ==== 93+0+10 (secure 0 0 0) 0x7f10d809aa60 con 0x7f10e0101660 2023-07-14T11:09:08.430 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:09:08.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 msgr2=0x7f10cc0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 0x7f10cc0a3600 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f10e0100b30 tx=0x7f10dc009260 comp rx=0 tx=0).stop 2023-07-14T11:09:08.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 msgr2=0x7f10e019f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f10d800d010 tx=0x7f10d800bc80 comp rx=0 tx=0).stop 2023-07-14T11:09:08.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 shutdown_connections 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10cc0a1150 0x7f10cc0a3600 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10e0101660 0x7f10e019f110 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10e00ffae0 0x7f10e0197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 --2- 172.21.15.40:0/4168619644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f10e0100cd0 0x7f10e01980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 >> 172.21.15.40:0/4168619644 conn(0x7f10e00fb0e0 msgr2=0x7f10e0106500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 shutdown_connections 2023-07-14T11:09:08.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.430+0000 7f10e83ff700 1 -- 172.21.15.40:0/4168619644 wait complete. 2023-07-14T11:09:08.451 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:08.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 -- 172.21.15.40:0/3341908813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 msgr2=0x7f0d1c0733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 --2- 172.21.15.40:0/3341908813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c0733e0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f0d10009990 tx=0x7f0d10009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 -- 172.21.15.40:0/3341908813 shutdown_connections 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 --2- 172.21.15.40:0/3341908813 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 --2- 172.21.15.40:0/3341908813 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 --2- 172.21.15.40:0/3341908813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c0733e0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 -- 172.21.15.40:0/3341908813 >> 172.21.15.40:0/3341908813 conn(0x7f0d1c0fb0e0 msgr2=0x7f0d1c0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 -- 172.21.15.40:0/3341908813 shutdown_connections 2023-07-14T11:09:08.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.652+0000 7f0d227ff700 1 -- 172.21.15.40:0/3341908813 wait complete. 2023-07-14T11:09:08.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.653+0000 7f0d227ff700 1 Processor -- start 2023-07-14T11:09:08.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 -- start start 2023-07-14T11:09:08.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c19f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d1c19f610 con 0x7f0d1c072f70 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d1c19f750 con 0x7f0d1c1079c0 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d227ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d1c19f8e0 con 0x7f0d1c074550 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54734/0 (socket says 172.21.15.40:54734) 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1bfff700 1 -- 172.21.15.40:0/2895760315 learned_addr learned my addr 172.21.15.40:0/2895760315 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:08.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1b7fe700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d20d9c700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c19f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1bfff700 1 -- 172.21.15.40:0/2895760315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 msgr2=0x7f0d1c197fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.654+0000 7f0d1bfff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c197fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d1bfff700 1 -- 172.21.15.40:0/2895760315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 msgr2=0x7f0d1c19f040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d1bfff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c19f040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d1bfff700 1 -- 172.21.15.40:0/2895760315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d14009c40 con 0x7f0d1c072f70 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d1b7fe700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:08.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d20d9c700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c19f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d1bfff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f0d0c00e970 tx=0x7f0d0c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d0c009790 con 0x7f0d1c072f70 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d0c00eec0 con 0x7f0d1c072f70 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d100095f0 con 0x7f0d1c072f70 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d0c005590 con 0x7f0d1c072f70 2023-07-14T11:09:08.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.655+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d1c19ff90 con 0x7f0d1c072f70 2023-07-14T11:09:08.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.656+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0d1c106070 con 0x7f0d1c072f70 2023-07-14T11:09:08.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.658+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f0d0c00cbd0 con 0x7f0d1c072f70 2023-07-14T11:09:08.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.660+0000 7f0d197fa700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 0x7f0d040a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:08.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.660+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f0d0c014070 con 0x7f0d1c072f70 2023-07-14T11:09:08.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.660+0000 7f0d1b7fe700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 0x7f0d040a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:08.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.661+0000 7f0d1b7fe700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 0x7f0d040a36d0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f0d1000b2e0 tx=0x7f0d10005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:08.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.663+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0d0c096250 con 0x7f0d1c072f70 2023-07-14T11:09:08.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.920+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f0d1c02c850 con 0x7f0d1c072f70 2023-07-14T11:09:08.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.920+0000 7f0d197fa700 1 -- 172.21.15.40:0/2895760315 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 1 v1) v1 ==== 93+0+501 (secure 0 0 0) 0x7f0d0c004ef0 con 0x7f0d1c072f70 2023-07-14T11:09:08.922 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:08.922 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2023-07-14T11:09:08.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 msgr2=0x7f0d040a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 0x7f0d040a36d0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f0d1000b2e0 tx=0x7f0d10005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 msgr2=0x7f0d1c197a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f0d0c00e970 tx=0x7f0d0c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 shutdown_connections 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0d040a1220 0x7f0d040a36d0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d1c1079c0 0x7f0d1c19f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d1c072f70 0x7f0d1c197a90 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 --2- 172.21.15.40:0/2895760315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d1c074550 0x7f0d1c197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 >> 172.21.15.40:0/2895760315 conn(0x7f0d1c0fb0e0 msgr2=0x7f0d1c0fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 shutdown_connections 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:08.923+0000 7f0d227ff700 1 -- 172.21.15.40:0/2895760315 wait complete. 2023-07-14T11:09:08.926 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 1 2023-07-14T11:09:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:08 smithi130 ceph-mon[129405]: Deploying daemon mds.l on smithi130 2023-07-14T11:09:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4215013464' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4168619644' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:09.043 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 systemd[1]: Starting Ceph mds.l for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:08 smithi045 ceph-mon[128791]: Deploying daemon mds.l on smithi130 2023-07-14T11:09:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4215013464' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4168619644' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:09.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:08 smithi040 ceph-mon[97954]: Deploying daemon mds.l on smithi130 2023-07-14T11:09:09.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4215013464' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:09.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4168619644' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:09.542 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 podman[154497]: 2023-07-14T11:09:10.042 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mds[154537]: -- 172.21.15.130:0/1120453710 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5616ba2cd1e0 con 0x5616ba2a2800 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: starting mds.l at 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 bash[154497]: e9db87968f868fee132ecff8074c1e156ab26e15b0143a15442b600cdc1e00fe 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:09:09 smithi130 systemd[1]: Started Ceph mds.l for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2895760315' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6832/2180852091,v1:172.21.15.130:6833/2180852091] up:boot 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: fsmap 1 up:standby 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:10.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:09 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:10.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2895760315' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6832/2180852091,v1:172.21.15.130:6833/2180852091] up:boot 2023-07-14T11:09:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: fsmap 1 up:standby 2023-07-14T11:09:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:09:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:09 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2895760315' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6832/2180852091,v1:172.21.15.130:6833/2180852091] up:boot 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: fsmap 1 up:standby 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:09:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.k", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:09 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:10.943 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:11.062 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:10 smithi045 ceph-mon[128791]: Deploying daemon mds.k on smithi045 2023-07-14T11:09:11.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 -- 172.21.15.40:0/3475037595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31241016a0 msgr2=0x7f3124107f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:11.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 --2- 172.21.15.40:0/3475037595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31241016a0 0x7f3124107f60 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f311c00b0a0 tx=0x7f311c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 -- 172.21.15.40:0/3475037595 shutdown_connections 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 --2- 172.21.15.40:0/3475037595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31241016a0 0x7f3124107f60 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 --2- 172.21.15.40:0/3475037595 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31240ffb20 0x7f31240fff30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 --2- 172.21.15.40:0/3475037595 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 0x7f3124101160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 -- 172.21.15.40:0/3475037595 >> 172.21.15.40:0/3475037595 conn(0x7f31240fb160 msgr2=0x7f31240fd570 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 -- 172.21.15.40:0/3475037595 shutdown_connections 2023-07-14T11:09:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.156+0000 7f312c1ff700 1 -- 172.21.15.40:0/3475037595 wait complete. 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.157+0000 7f312c1ff700 1 Processor -- start 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.157+0000 7f312c1ff700 1 -- start start 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 0x7f3124198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 0x7f312419f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f312419f650 con 0x7f31240ffb20 2023-07-14T11:09:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f312419f790 con 0x7f3124100d10 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312c1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f312419f920 con 0x7f31241016a0 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f3129f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f3129f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54752/0 (socket says 172.21.15.40:54752) 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 0x7f3124198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f3129f9b700 1 -- 172.21.15.40:0/700174326 learned_addr learned my addr 172.21.15.40:0/700174326 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.158+0000 7f312a79c700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 0x7f312419f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 -- 172.21.15.40:0/700174326 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 msgr2=0x7f312419f080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 0x7f312419f080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 -- 172.21.15.40:0/700174326 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 msgr2=0x7f3124198010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 0x7f3124198010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 -- 172.21.15.40:0/700174326 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f311c009d00 con 0x7f31240ffb20 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f312a79c700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 0x7f312419f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f3129f9b700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f312000e970 tx=0x7f312000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3120009790 con 0x7f31240ffb20 2023-07-14T11:09:11.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f312419fc00 con 0x7f31240ffb20 2023-07-14T11:09:11.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f312000eec0 con 0x7f31240ffb20 2023-07-14T11:09:11.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3120003cf0 con 0x7f31240ffb20 2023-07-14T11:09:11.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.159+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31241a0110 con 0x7f31240ffb20 2023-07-14T11:09:11.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.160+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f312410b300 con 0x7f31240ffb20 2023-07-14T11:09:11.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.162+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f312000ccd0 con 0x7f31240ffb20 2023-07-14T11:09:11.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.164+0000 7f311affd700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 0x7f31100a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:11.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.165+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f3120014070 con 0x7f31240ffb20 2023-07-14T11:09:11.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.165+0000 7f312979a700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 0x7f31100a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:11.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.165+0000 7f312979a700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 0x7f31100a3650 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f3114007be0 tx=0x7f31140073d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:11.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.169+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f31200962b0 con 0x7f31240ffb20 2023-07-14T11:09:11.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:10 smithi130 ceph-mon[129405]: Deploying daemon mds.k on smithi045 2023-07-14T11:09:11.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:10 smithi040 ceph-mon[97954]: Deploying daemon mds.k on smithi045 2023-07-14T11:09:11.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.424+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f31241a08f0 con 0x7f31240ffb20 2023-07-14T11:09:11.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.425+0000 7f311affd700 1 -- 172.21.15.40:0/700174326 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 2 v2) v1 ==== 93+0+1260 (secure 0 0 0) 0x7f31200998d0 con 0x7f31240ffb20 2023-07-14T11:09:11.427 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:11.427 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":2,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 msgr2=0x7f31100a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 0x7f31100a3650 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f3114007be0 tx=0x7f31140073d0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 msgr2=0x7f3124197ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f312000e970 tx=0x7f312000ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 shutdown_connections 2023-07-14T11:09:11.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f31100a11a0 0x7f31100a3650 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f31241016a0 0x7f312419f080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f31240ffb20 0x7f3124197ad0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 --2- 172.21.15.40:0/700174326 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3124100d10 0x7f3124198010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 >> 172.21.15.40:0/700174326 conn(0x7f31240fb160 msgr2=0x7f31240fd240 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 shutdown_connections 2023-07-14T11:09:11.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:11.427+0000 7f312c1ff700 1 -- 172.21.15.40:0/700174326 wait complete. 2023-07-14T11:09:11.431 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 2 2023-07-14T11:09:11.792 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:11 smithi045 systemd[1]: Starting Ceph mds.k for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:12.048 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:11 smithi045 ceph-mon[128791]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:12.049 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/700174326' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:12.049 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:11 smithi045 podman[153113]: 2023-07-14T11:09:12.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:11 smithi130 ceph-mon[129405]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:12.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/700174326' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:12.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:11 smithi040 ceph-mon[97954]: pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:12.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/700174326' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:12.313 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:12 smithi045 ceph-mds[153153]: -- 172.21.15.45:0/2990399959 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ccabf151e0 con 0x55ccabeea800 2023-07-14T11:09:12.314 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-k[153129]: starting mds.k at 2023-07-14T11:09:12.314 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:12 smithi045 bash[153113]: 9b13c4cc289f6c0601b21c8de352ab86552248be3784f2cc91f5fa20d5c0f70c 2023-07-14T11:09:12.314 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:09:12 smithi045 systemd[1]: Started Ceph mds.k for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:13.397 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.397 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.397 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.397 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:13.397 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: Deploying daemon mds.j on smithi040 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6832/2674461503,v1:172.21.15.45:6833/2674461503] up:boot 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: fsmap 2 up:standby 2023-07-14T11:09:13.398 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:13 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:09:13.449 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:13.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: Deploying daemon mds.j on smithi040 2023-07-14T11:09:13.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:13.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6832/2674461503,v1:172.21.15.45:6833/2674461503] up:boot 2023-07-14T11:09:13.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: fsmap 2 up:standby 2023-07-14T11:09:13.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:13 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:09:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:13.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.j", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: Deploying daemon mds.j on smithi040 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6832/2674461503,v1:172.21.15.45:6833/2674461503] up:boot 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: fsmap 2 up:standby 2023-07-14T11:09:13.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:13 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:09:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/1187980977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 msgr2=0x7f4e04101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/1187980977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04101140 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f4df4009990 tx=0x7f4df4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/1187980977 shutdown_connections 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/1187980977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e04101680 0x7f4e04107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/1187980977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e040ffb00 0x7f4e040fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/1187980977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04101140 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/1187980977 >> 172.21.15.40:0/1187980977 conn(0x7f4e040fb0e0 msgr2=0x7f4e040fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/1187980977 shutdown_connections 2023-07-14T11:09:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.658+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/1187980977 wait complete. 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.659+0000 7f4e0c5ff700 1 Processor -- start 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.659+0000 7f4e0c5ff700 1 -- start start 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.659+0000 7f4e0c5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e040ffb00 0x7f4e041979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.659+0000 7f4e0c5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.659+0000 7f4e0c5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e04101680 0x7f4e0419efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e0c5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e0419f570 con 0x7f4e04100cf0 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e0c5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e0419f6b0 con 0x7f4e040ffb00 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e0c5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e0419f840 con 0x7f4e04101680 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49718/0 (socket says 172.21.15.40:49718) 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 -- 172.21.15.40:0/2955892527 learned_addr learned my addr 172.21.15.40:0/2955892527 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e0a39b700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e040ffb00 0x7f4e041979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e0ab9c700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e04101680 0x7f4e0419efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 -- 172.21.15.40:0/2955892527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e04101680 msgr2=0x7f4e0419efa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e04101680 0x7f4e0419efa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 -- 172.21.15.40:0/2955892527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e040ffb00 msgr2=0x7f4e041979f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e040ffb00 0x7f4e041979f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 -- 172.21.15.40:0/2955892527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4df40095f0 con 0x7f4e04100cf0 2023-07-14T11:09:13.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.660+0000 7f4e09b9a700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4df400c010 tx=0x7f4df4004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.661+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4df401d070 con 0x7f4e04100cf0 2023-07-14T11:09:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.661+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4df4022470 con 0x7f4e04100cf0 2023-07-14T11:09:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.661+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e0419fac0 con 0x7f4e04100cf0 2023-07-14T11:09:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.661+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4df4017650 con 0x7f4e04100cf0 2023-07-14T11:09:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.661+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e0419ff40 con 0x7f4e04100cf0 2023-07-14T11:09:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.662+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e0410b320 con 0x7f4e04100cf0 2023-07-14T11:09:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.664+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4df40225e0 con 0x7f4e04100cf0 2023-07-14T11:09:13.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.665+0000 7f4dfb7fe700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 0x7f4df00a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:13.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.666+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f4df40cd990 con 0x7f4e04100cf0 2023-07-14T11:09:13.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.666+0000 7f4e0a39b700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 0x7f4df00a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.666+0000 7f4e0a39b700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 0x7f4df00a36d0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f4e04100b50 tx=0x7f4e00009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:13.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.670+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4df4097570 con 0x7f4e04100cf0 2023-07-14T11:09:13.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.955+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4e041a04f0 con 0x7f4e04100cf0 2023-07-14T11:09:13.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.956+0000 7f4dfb7fe700 1 -- 172.21.15.40:0/2955892527 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 3 v3) v1 ==== 93+0+2017 (secure 0 0 0) 0x7f4df409ab90 con 0x7f4e04100cf0 2023-07-14T11:09:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:13.958 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2023-07-14T11:09:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 msgr2=0x7f4df00a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 0x7f4df00a36d0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f4e04100b50 tx=0x7f4e00009260 comp rx=0 tx=0).stop 2023-07-14T11:09:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 msgr2=0x7f4e04197f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4df400c010 tx=0x7f4df4004730 comp rx=0 tx=0).stop 2023-07-14T11:09:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 shutdown_connections 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4df00a1220 0x7f4df00a36d0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4e04101680 0x7f4e0419efa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4e040ffb00 0x7f4e041979f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 --2- 172.21.15.40:0/2955892527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4e04100cf0 0x7f4e04197f30 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 >> 172.21.15.40:0/2955892527 conn(0x7f4e040fb0e0 msgr2=0x7f4e040fd260 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 shutdown_connections 2023-07-14T11:09:13.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:13.958+0000 7f4e0c5ff700 1 -- 172.21.15.40:0/2955892527 wait complete. 2023-07-14T11:09:13.962 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 3 2023-07-14T11:09:14.415 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:14 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2955892527' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:14.416 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:14 smithi040 systemd[1]: Starting Ceph mds.j for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:14.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:14 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2955892527' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:14.563 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:14 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2955892527' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:15.176 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:14 smithi040 podman[130773]: 2023-07-14T11:09:15.176 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:15 smithi040 bash[130773]: 85248e8472bfd30046377a42c6346e315cf57141973e68d2d07cf8ffc4c25719 2023-07-14T11:09:15.177 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mds[130813]: -- 172.21.15.40:0/1982618568 <== mon.0 v2:172.21.15.40:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555dbd76d1e0 con 0x555dbd742c00 2023-07-14T11:09:15.177 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-j[130789]: starting mds.j at 2023-07-14T11:09:15.546 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:09:15 smithi040 systemd[1]: Started Ceph mds.j for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:15.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:15.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:15.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:15.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:15 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:15 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.d", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:15 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:15.980 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:16.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 -- 172.21.15.40:0/3546242821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 msgr2=0x7f97f80fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:16.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 --2- 172.21.15.40:0/3546242821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80fff10 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f97f4009990 tx=0x7f97f4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 -- 172.21.15.40:0/3546242821 shutdown_connections 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 --2- 172.21.15.40:0/3546242821 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 0x7f97f8107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 --2- 172.21.15.40:0/3546242821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80fff10 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 --2- 172.21.15.40:0/3546242821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 0x7f97f8101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 -- 172.21.15.40:0/3546242821 >> 172.21.15.40:0/3546242821 conn(0x7f97f80fb0e0 msgr2=0x7f97f80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 -- 172.21.15.40:0/3546242821 shutdown_connections 2023-07-14T11:09:16.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.190+0000 7f98003ff700 1 -- 172.21.15.40:0/3546242821 wait complete. 2023-07-14T11:09:16.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.191+0000 7f98003ff700 1 Processor -- start 2023-07-14T11:09:16.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.191+0000 7f98003ff700 1 -- start start 2023-07-14T11:09:16.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:16.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 0x7f97f8072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 0x7f97f8072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f80735e0 con 0x7f97f80ffb00 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f8073750 con 0x7f97f8101680 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f98003ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f97f819f8b0 con 0x7f97f8100cf0 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49760/0 (socket says 172.21.15.40:49760) 2023-07-14T11:09:16.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 -- 172.21.15.40:0/1691090387 learned_addr learned my addr 172.21.15.40:0/1691090387 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fd99a700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 0x7f97f8072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe99c700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 0x7f97f8072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 -- 172.21.15.40:0/1691090387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 msgr2=0x7f97f8072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 0x7f97f8072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 -- 172.21.15.40:0/1691090387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 msgr2=0x7f97f8072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 0x7f97f8072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.192+0000 7f97fe19b700 1 -- 172.21.15.40:0/1691090387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f97f40095f0 con 0x7f97f80ffb00 2023-07-14T11:09:16.197 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f97fe19b700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f97f4000c00 tx=0x7f97f4005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:16.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f401d070 con 0x7f97f80ffb00 2023-07-14T11:09:16.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f97f4003f60 con 0x7f97f80ffb00 2023-07-14T11:09:16.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f97f4022c70 con 0x7f97f80ffb00 2023-07-14T11:09:16.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f97f819fb30 con 0x7f97f80ffb00 2023-07-14T11:09:16.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.193+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f97f819ff00 con 0x7f97f80ffb00 2023-07-14T11:09:16.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.194+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f97f810b320 con 0x7f97f80ffb00 2023-07-14T11:09:16.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.196+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f97f40040d0 con 0x7f97f80ffb00 2023-07-14T11:09:16.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.199+0000 7f97ef7fe700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 0x7f97e40a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:16.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.199+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f97f40ce260 con 0x7f97f80ffb00 2023-07-14T11:09:16.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.199+0000 7f97fd99a700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 0x7f97e40a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:16.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.200+0000 7f97fd99a700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 0x7f97e40a3650 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f97e8007c60 tx=0x7f97e80073d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:16.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.202+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f97f4097e40 con 0x7f97f80ffb00 2023-07-14T11:09:16.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.488+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f97f81069b0 con 0x7f97f80ffb00 2023-07-14T11:09:16.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.488+0000 7f97ef7fe700 1 -- 172.21.15.40:0/1691090387 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 4 v4) v1 ==== 93+0+2774 (secure 0 0 0) 0x7f97f409b460 con 0x7f97f80ffb00 2023-07-14T11:09:16.490 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:16.491 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":4,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[]} 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 msgr2=0x7f97e40a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 0x7f97e40a3650 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f97e8007c60 tx=0x7f97e80073d0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 msgr2=0x7f97f80742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f97f4000c00 tx=0x7f97f4005740 comp rx=0 tx=0).stop 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 shutdown_connections 2023-07-14T11:09:16.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f97e40a11a0 0x7f97e40a3650 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f97f8101680 0x7f97f8072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f97f80ffb00 0x7f97f80742c0 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 --2- 172.21.15.40:0/1691090387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f97f8100cf0 0x7f97f8072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 >> 172.21.15.40:0/1691090387 conn(0x7f97f80fb0e0 msgr2=0x7f97f80fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 shutdown_connections 2023-07-14T11:09:16.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:16.491+0000 7f98003ff700 1 -- 172.21.15.40:0/1691090387 wait complete. 2023-07-14T11:09:16.495 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 4 2023-07-14T11:09:16.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:16 smithi130 ceph-mon[129405]: Deploying daemon mds.d on smithi040 2023-07-14T11:09:16.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:16 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6834/2807710685,v1:172.21.15.40:6835/2807710685] up:boot 2023-07-14T11:09:16.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:16 smithi130 ceph-mon[129405]: fsmap 3 up:standby 2023-07-14T11:09:16.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:16 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:09:16.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:16 smithi040 ceph-mon[97954]: Deploying daemon mds.d on smithi040 2023-07-14T11:09:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:16 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6834/2807710685,v1:172.21.15.40:6835/2807710685] up:boot 2023-07-14T11:09:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:16 smithi040 ceph-mon[97954]: fsmap 3 up:standby 2023-07-14T11:09:16.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:16 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:09:16.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:16 smithi045 ceph-mon[128791]: Deploying daemon mds.d on smithi040 2023-07-14T11:09:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:16 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6834/2807710685,v1:172.21.15.40:6835/2807710685] up:boot 2023-07-14T11:09:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:16 smithi045 ceph-mon[128791]: fsmap 3 up:standby 2023-07-14T11:09:16.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:16 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:09:17.625 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:17 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1691090387' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:17.626 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:17 smithi040 ceph-mon[97954]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:17.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:17 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1691090387' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:17 smithi130 ceph-mon[129405]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:17.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:17 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1691090387' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:17 smithi045 ceph-mon[128791]: pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:18.046 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:17 smithi040 systemd[1]: Starting Ceph mds.d for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:18.370 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:18 smithi040 podman[131162]: 2023-07-14T11:09:18.370 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:18 smithi040 bash[131162]: 3db246c6c8ae572d50197e27d3e6793fc9d1b9dddd7e62c463984f4d6bf8ee8d 2023-07-14T11:09:18.513 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:18.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 -- 172.21.15.40:0/3262696789 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf70072ef0 msgr2=0x7fdf70073360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 --2- 172.21.15.40:0/3262696789 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf70072ef0 0x7fdf70073360 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fdf5c009990 tx=0x7fdf5c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 -- 172.21.15.40:0/3262696789 shutdown_connections 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 --2- 172.21.15.40:0/3262696789 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf701079b0 0x7fdf70109e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 --2- 172.21.15.40:0/3262696789 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70072920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 --2- 172.21.15.40:0/3262696789 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf70072ef0 0x7fdf70073360 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 -- 172.21.15.40:0/3262696789 >> 172.21.15.40:0/3262696789 conn(0x7fdf700fb0e0 msgr2=0x7fdf700fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:18.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 -- 172.21.15.40:0/3262696789 shutdown_connections 2023-07-14T11:09:18.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.722+0000 7fdf76fff700 1 -- 172.21.15.40:0/3262696789 wait complete. 2023-07-14T11:09:18.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.723+0000 7fdf76fff700 1 Processor -- start 2023-07-14T11:09:18.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 -- start start 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf70072ef0 0x7fdf70197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 0x7fdf7019efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf7019f590 con 0x7fdf700744d0 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf7019f6d0 con 0x7fdf701079b0 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf76fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf7019f860 con 0x7fdf70072ef0 2023-07-14T11:09:18.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49808/0 (socket says 172.21.15.40:49808) 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf6ffff700 1 -- 172.21.15.40:0/2550798444 learned_addr learned my addr 172.21.15.40:0/2550798444 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf7559c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 0x7fdf7019efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.724+0000 7fdf74d9b700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf70072ef0 0x7fdf70197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 -- 172.21.15.40:0/2550798444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf70072ef0 msgr2=0x7fdf70197a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf70072ef0 0x7fdf70197a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 -- 172.21.15.40:0/2550798444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 msgr2=0x7fdf7019efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 0x7fdf7019efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 -- 172.21.15.40:0/2550798444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdf5c0095f0 con 0x7fdf700744d0 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf7559c700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 0x7fdf7019efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6ffff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fdf5c00b680 tx=0x7fdf5c005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf5c01d070 con 0x7fdf700744d0 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdf7019fae0 con 0x7fdf700744d0 2023-07-14T11:09:18.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdf5c022470 con 0x7fdf700744d0 2023-07-14T11:09:18.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf5c017650 con 0x7fdf700744d0 2023-07-14T11:09:18.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.725+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdf7019ff10 con 0x7fdf700744d0 2023-07-14T11:09:18.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.726+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdf701060f0 con 0x7fdf700744d0 2023-07-14T11:09:18.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.728+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fdf5c0225e0 con 0x7fdf700744d0 2023-07-14T11:09:18.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.730+0000 7fdf6dffb700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 0x7fdf600a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:18.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.730+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fdf5c0ce060 con 0x7fdf700744d0 2023-07-14T11:09:18.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.730+0000 7fdf74d9b700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 0x7fdf600a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:18.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.730+0000 7fdf74d9b700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 0x7fdf600a3650 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fdf64009c60 tx=0x7fdf64009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:18.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.732+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdf5c097c40 con 0x7fdf700744d0 2023-07-14T11:09:18.796 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:18 smithi040 ceph-mds[131202]: -- 172.21.15.40:0/3776966629 <== mon.0 v2:172.21.15.40:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563c203a51e0 con 0x563c2037a800 2023-07-14T11:09:18.796 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: starting mds.d at 2023-07-14T11:09:18.796 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:09:18 smithi040 systemd[1]: Started Ceph mds.d for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:18.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.986+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fdf7002c850 con 0x7fdf700744d0 2023-07-14T11:09:18.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.987+0000 7fdf6dffb700 1 -- 172.21.15.40:0/2550798444 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) v1 ==== 93+0+3528 (secure 0 0 0) 0x7fdf5c027de0 con 0x7fdf700744d0 2023-07-14T11:09:18.989 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:18.989 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5}],"filesystems":[]} 2023-07-14T11:09:18.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 msgr2=0x7fdf600a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:18.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 0x7fdf600a3650 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fdf64009c60 tx=0x7fdf64009260 comp rx=0 tx=0).stop 2023-07-14T11:09:18.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 msgr2=0x7fdf70197f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7fdf5c00b680 tx=0x7fdf5c005740 comp rx=0 tx=0).stop 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 shutdown_connections 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf600a11a0 0x7fdf600a3650 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf701079b0 0x7fdf7019efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf70072ef0 0x7fdf70197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 --2- 172.21.15.40:0/2550798444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf700744d0 0x7fdf70197f50 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.989+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 >> 172.21.15.40:0/2550798444 conn(0x7fdf700fb0e0 msgr2=0x7fdf700fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:18.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.990+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 shutdown_connections 2023-07-14T11:09:18.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:18.990+0000 7fdf76fff700 1 -- 172.21.15.40:0/2550798444 wait complete. 2023-07-14T11:09:18.993 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 5 2023-07-14T11:09:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6836/625677299,v1:172.21.15.40:6837/625677299] up:boot 2023-07-14T11:09:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: fsmap 4 up:standby 2023-07-14T11:09:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:09:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: Deploying daemon mds.e on smithi045 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:19.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:19 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2550798444' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:19.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6836/625677299,v1:172.21.15.40:6837/625677299] up:boot 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: fsmap 4 up:standby 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:19.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:19.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: Deploying daemon mds.e on smithi045 2023-07-14T11:09:19.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:19.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:19 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2550798444' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6836/625677299,v1:172.21.15.40:6837/625677299] up:boot 2023-07-14T11:09:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: fsmap 4 up:standby 2023-07-14T11:09:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:09:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.e", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: Deploying daemon mds.e on smithi045 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:19 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2550798444' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:20.509 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 systemd[1]: Starting Ceph mds.e for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:20.509 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 podman[153457]: 2023-07-14T11:09:20.774 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 ceph-mds[153497]: -- 172.21.15.45:0/1525730242 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55674fa931e0 con 0x55674fa68c00 2023-07-14T11:09:20.775 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: starting mds.e at 2023-07-14T11:09:21.011 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:21.064 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 bash[153457]: 0d862ba0764bb631225f9c72afe2e1b1804f33cb5f8415ed9cb028d11fd8c4a0 2023-07-14T11:09:21.064 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:09:20 smithi045 systemd[1]: Started Ceph mds.e for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:21.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.223+0000 7f03853ff700 1 -- 172.21.15.40:0/1873646208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 msgr2=0x7f0380101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.223+0000 7f03853ff700 1 --2- 172.21.15.40:0/1873646208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380101140 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f0374009990 tx=0x7f0374009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.223+0000 7f03853ff700 1 -- 172.21.15.40:0/1873646208 shutdown_connections 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.223+0000 7f03853ff700 1 --2- 172.21.15.40:0/1873646208 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.224+0000 7f03853ff700 1 --2- 172.21.15.40:0/1873646208 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 0x7f03800fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.224+0000 7f03853ff700 1 --2- 172.21.15.40:0/1873646208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380101140 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.224+0000 7f03853ff700 1 -- 172.21.15.40:0/1873646208 >> 172.21.15.40:0/1873646208 conn(0x7f03800fb0e0 msgr2=0x7f03800fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:21.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.224+0000 7f03853ff700 1 -- 172.21.15.40:0/1873646208 shutdown_connections 2023-07-14T11:09:21.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.224+0000 7f03853ff700 1 -- 172.21.15.40:0/1873646208 wait complete. 2023-07-14T11:09:21.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 Processor -- start 2023-07-14T11:09:21.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 -- start start 2023-07-14T11:09:21.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 0x7f0380074270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:21.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:21.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03800735e0 con 0x7f0380101680 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0380073750 con 0x7f0380100cf0 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f03853ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f038019f830 con 0x7f03800ffb00 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f037f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.225+0000 7f037effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 0x7f0380074270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:49826/0 (socket says 172.21.15.40:49826) 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037e7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:21.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 -- 172.21.15.40:0/2272922719 learned_addr learned my addr 172.21.15.40:0/2272922719 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 -- 172.21.15.40:0/2272922719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 msgr2=0x7f0380074270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 0x7f0380074270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 -- 172.21.15.40:0/2272922719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 msgr2=0x7f0380072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 -- 172.21.15.40:0/2272922719 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0378009c40 con 0x7f0380101680 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f037f7fe700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f037800c370 tx=0x7f037800c680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:21.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0378017070 con 0x7f0380101680 2023-07-14T11:09:21.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03740095f0 con 0x7f0380101680 2023-07-14T11:09:21.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f037800f040 con 0x7f0380101680 2023-07-14T11:09:21.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.226+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f037800e050 con 0x7f0380101680 2023-07-14T11:09:21.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.227+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f038019fe30 con 0x7f0380101680 2023-07-14T11:09:21.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.228+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0380106fd0 con 0x7f0380101680 2023-07-14T11:09:21.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.230+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f037800cb00 con 0x7f0380101680 2023-07-14T11:09:21.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.231+0000 7f0367fff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 0x7f03680a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:21.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.231+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f03780cc410 con 0x7f0380101680 2023-07-14T11:09:21.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.231+0000 7f037effd700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 0x7f03680a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:21.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.232+0000 7f037effd700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 0x7f03680a36d0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f0380100b50 tx=0x7f037000a3b0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:21.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.233+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0378095f60 con 0x7f0380101680 2023-07-14T11:09:21.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.487+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f038002c850 con 0x7f0380101680 2023-07-14T11:09:21.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.487+0000 7f0367fff700 1 -- 172.21.15.40:0/2272922719 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 5 v5) v1 ==== 93+0+3528 (secure 0 0 0) 0x7f0378099580 con 0x7f0380101680 2023-07-14T11:09:21.490 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:21.491 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":5,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5}],"filesystems":[]} 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 msgr2=0x7f03680a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 0x7f03680a36d0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f0380100b50 tx=0x7f037000a3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 msgr2=0x7f0380072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7f037800c370 tx=0x7f037800c680 comp rx=0 tx=0).stop 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 shutdown_connections 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f03680a1220 0x7f03680a36d0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0380101680 0x7f0380072e40 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f03800ffb00 0x7f0380074270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 --2- 172.21.15.40:0/2272922719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0380100cf0 0x7f0380072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 >> 172.21.15.40:0/2272922719 conn(0x7f03800fb0e0 msgr2=0x7f0380104110 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 shutdown_connections 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:21.490+0000 7f03853ff700 1 -- 172.21.15.40:0/2272922719 wait complete. 2023-07-14T11:09:21.494 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 5 2023-07-14T11:09:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: Deploying daemon mds.f on smithi130 2023-07-14T11:09:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2272922719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6834/1717726567,v1:172.21.15.45:6835/1717726567] up:boot 2023-07-14T11:09:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: fsmap 5 up:standby 2023-07-14T11:09:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:21 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:09:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: Deploying daemon mds.f on smithi130 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2272922719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6834/1717726567,v1:172.21.15.45:6835/1717726567] up:boot 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: fsmap 5 up:standby 2023-07-14T11:09:22.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:21 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:09:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.f", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: Deploying daemon mds.f on smithi130 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2272922719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6834/1717726567,v1:172.21.15.45:6835/1717726567] up:boot 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: fsmap 5 up:standby 2023-07-14T11:09:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:21 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:09:22.543 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 systemd[1]: Starting Ceph mds.f for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:22.887 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 podman[154843]: 2023-07-14T11:09:23.292 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 bash[154843]: 748a9c09cc2bdbe1d76a76875035d2fae62a80fedf962f600e960c88e6ba1109 2023-07-14T11:09:23.293 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 ceph-mds[154883]: -- 172.21.15.130:0/3152733911 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56503d22d1e0 con 0x56503d203000 2023-07-14T11:09:23.293 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: starting mds.f at 2023-07-14T11:09:23.293 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:09:22 smithi130 systemd[1]: Started Ceph mds.f for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:23.512 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:23.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 -- 172.21.15.40:0/2658665100 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1284100cf0 msgr2=0x7f1284101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:23.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 --2- 172.21.15.40:0/2658665100 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1284100cf0 0x7f1284101140 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f1278009990 tx=0x7f1278009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 -- 172.21.15.40:0/2658665100 shutdown_connections 2023-07-14T11:09:23.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 --2- 172.21.15.40:0/2658665100 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284101680 0x7f1284107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 --2- 172.21.15.40:0/2658665100 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12840ffb00 0x7f12840fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 --2- 172.21.15.40:0/2658665100 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1284100cf0 0x7f1284101140 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 -- 172.21.15.40:0/2658665100 >> 172.21.15.40:0/2658665100 conn(0x7f12840fb0e0 msgr2=0x7f12840fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:23.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 -- 172.21.15.40:0/2658665100 shutdown_connections 2023-07-14T11:09:23.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.724+0000 7f128a9ff700 1 -- 172.21.15.40:0/2658665100 wait complete. 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.725+0000 7f128a9ff700 1 Processor -- start 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.725+0000 7f128a9ff700 1 -- start start 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284100cf0 0x7f1284197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1284101680 0x7f128419efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:23.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f128419f5a0 con 0x7f12840ffb00 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f128419f6e0 con 0x7f1284101680 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f128a9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f128419f870 con 0x7f1284100cf0 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:44542/0 (socket says 172.21.15.40:44542) 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 -- 172.21.15.40:0/391983829 learned_addr learned my addr 172.21.15.40:0/391983829 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1288f9c700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1284101680 0x7f128419efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:23.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f12837fe700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284100cf0 0x7f1284197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 -- 172.21.15.40:0/391983829 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284100cf0 msgr2=0x7f1284197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284100cf0 0x7f1284197f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 -- 172.21.15.40:0/391983829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1284101680 msgr2=0x7f128419efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1284101680 0x7f128419efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.726+0000 7f1283fff700 1 -- 172.21.15.40:0/391983829 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f12780095f0 con 0x7f12840ffb00 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f1283fff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f127400e970 tx=0x7f127400ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1274009790 con 0x7f12840ffb00 2023-07-14T11:09:23.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f127400c7e0 con 0x7f12840ffb00 2023-07-14T11:09:23.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1274003c70 con 0x7f12840ffb00 2023-07-14T11:09:23.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f128419fb50 con 0x7f12840ffb00 2023-07-14T11:09:23.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.727+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12841a0060 con 0x7f12840ffb00 2023-07-14T11:09:23.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.728+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f128410b320 con 0x7f12840ffb00 2023-07-14T11:09:23.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.730+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f127400cc50 con 0x7f12840ffb00 2023-07-14T11:09:23.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.731+0000 7f12817fa700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 0x7f126c0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:23.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.732+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f1274014070 con 0x7f12840ffb00 2023-07-14T11:09:23.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.732+0000 7f12837fe700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 0x7f126c0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:23.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.732+0000 7f12837fe700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 0x7f126c0a36d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f127800b2e0 tx=0x7f1278005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:23.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.735+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1274096170 con 0x7f12840ffb00 2023-07-14T11:09:23.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.992+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f128410b510 con 0x7f12840ffb00 2023-07-14T11:09:23.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.994+0000 7f12817fa700 1 -- 172.21.15.40:0/391983829 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 7 v7) v1 ==== 93+0+5045 (secure 0 0 0) 0x7f1274003dd0 con 0x7f12840ffb00 2023-07-14T11:09:23.997 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:23.997 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":7,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7}],"filesystems":[]} 2023-07-14T11:09:23.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 msgr2=0x7f126c0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:23.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 0x7f126c0a36d0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f127800b2e0 tx=0x7f1278005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 msgr2=0x7f1284197a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f127400e970 tx=0x7f127400ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 shutdown_connections 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f126c0a1220 0x7f126c0a36d0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1284101680 0x7f128419efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f12840ffb00 0x7f1284197a20 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:24.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 --2- 172.21.15.40:0/391983829 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1284100cf0 0x7f1284197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:24.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.996+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 >> 172.21.15.40:0/391983829 conn(0x7f12840fb0e0 msgr2=0x7f1284106590 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:24.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.997+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 shutdown_connections 2023-07-14T11:09:24.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:23.997+0000 7f128a9ff700 1 -- 172.21.15.40:0/391983829 wait complete. 2023-07-14T11:09:24.001 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 7 2023-07-14T11:09:24.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:24.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.288 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:boot 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: fsmap 6 up:standby 2023-07-14T11:09:24.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:23 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:09:24.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:boot 2023-07-14T11:09:24.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: fsmap 6 up:standby 2023-07-14T11:09:24.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:23 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:09:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.c", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:boot 2023-07-14T11:09:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: fsmap 6 up:standby 2023-07-14T11:09:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:23 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:09:24.542 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:24 smithi130 systemd[1]: Starting Ceph mds.c for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:25.069 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:24 smithi130 ceph-mon[129405]: Deploying daemon mds.c on smithi130 2023-07-14T11:09:25.070 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:24 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/391983829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:25.070 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:24 smithi130 podman[155184]: 2023-07-14T11:09:25.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:24 smithi040 ceph-mon[97954]: Deploying daemon mds.c on smithi130 2023-07-14T11:09:25.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:24 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/391983829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:24 smithi045 ceph-mon[128791]: Deploying daemon mds.c on smithi130 2023-07-14T11:09:25.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:24 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/391983829' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:25.542 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:25 smithi130 ceph-mds[155224]: -- 172.21.15.130:0/2751641786 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558d6541b1e0 con 0x558d653f1000 2023-07-14T11:09:25.543 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:25 smithi130 bash[155184]: 32e5999b480ed276add4e0ffa0d7ee5ba34948fc034e7bef6b45d0fa5a7be7c6 2023-07-14T11:09:25.543 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: starting mds.c at 2023-07-14T11:09:25.543 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:09:25 smithi130 systemd[1]: Started Ceph mds.c for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:26.019 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:26 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:26.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 -- 172.21.15.40:0/4148236829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc100cf0 msgr2=0x7f29cc101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 --2- 172.21.15.40:0/4148236829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc100cf0 0x7f29cc101140 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f29b8009990 tx=0x7f29b8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 -- 172.21.15.40:0/4148236829 shutdown_connections 2023-07-14T11:09:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 --2- 172.21.15.40:0/4148236829 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc101680 0x7f29cc107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 --2- 172.21.15.40:0/4148236829 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 0x7f29cc0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 --2- 172.21.15.40:0/4148236829 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc100cf0 0x7f29cc101140 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 -- 172.21.15.40:0/4148236829 >> 172.21.15.40:0/4148236829 conn(0x7f29cc0fb0e0 msgr2=0x7f29cc0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 -- 172.21.15.40:0/4148236829 shutdown_connections 2023-07-14T11:09:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.230+0000 7f29d2dff700 1 -- 172.21.15.40:0/4148236829 wait complete. 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 Processor -- start 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 -- start start 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 0x7f29cc197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc100cf0 0x7f29cc197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:26.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29cc19f5d0 con 0x7f29cc100cf0 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29cc19f710 con 0x7f29cc101680 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.231+0000 7f29d2dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f29cc19f8a0 con 0x7f29cc0ffb00 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39784/0 (socket says 172.21.15.40:39784) 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 -- 172.21.15.40:0/3496452987 learned_addr learned my addr 172.21.15.40:0/3496452987 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:26.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29cbfff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc100cf0 0x7f29cc197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d0b9b700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 0x7f29cc197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 -- 172.21.15.40:0/3496452987 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 msgr2=0x7f29cc197a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 0x7f29cc197a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 -- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc100cf0 msgr2=0x7f29cc197f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc100cf0 0x7f29cc197f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 -- 172.21.15.40:0/3496452987 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f29c4009c40 con 0x7f29cc101680 2023-07-14T11:09:26.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29d139c700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f29c400c370 tx=0x7f29c400c680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:26.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29c4017070 con 0x7f29cc101680 2023-07-14T11:09:26.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.232+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f29c400f040 con 0x7f29cc101680 2023-07-14T11:09:26.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.233+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f29b80095f0 con 0x7f29cc101680 2023-07-14T11:09:26.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.233+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f29c400e050 con 0x7f29cc101680 2023-07-14T11:09:26.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.233+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f29cc19fef0 con 0x7f29cc101680 2023-07-14T11:09:26.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.234+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f29cc10b320 con 0x7f29cc101680 2023-07-14T11:09:26.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.236+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f29c4004bd0 con 0x7f29cc101680 2023-07-14T11:09:26.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.237+0000 7f29c9ffb700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 0x7f29bc0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:26.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.237+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f29c40cbd80 con 0x7f29cc101680 2023-07-14T11:09:26.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.237+0000 7f29d0b9b700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 0x7f29bc0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:26.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.238+0000 7f29d0b9b700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 0x7f29bc0a36d0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f29cc100b50 tx=0x7f29c0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:26.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.239+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f29c40958d0 con 0x7f29cc101680 2023-07-14T11:09:26.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:26.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:26 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:26.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:26.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.h", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:26.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:26 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:26.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.492+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f29cc10b510 con 0x7f29cc101680 2023-07-14T11:09:26.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.493+0000 7f29c9ffb700 1 -- 172.21.15.40:0/3496452987 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 8 v8) v1 ==== 93+0+5805 (secure 0 0 0) 0x7f29c4098ef0 con 0x7f29cc101680 2023-07-14T11:09:26.496 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:26.496 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":8,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8}],"filesystems":[]} 2023-07-14T11:09:26.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 msgr2=0x7f29bc0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:26.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 0x7f29bc0a36d0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f29cc100b50 tx=0x7f29c0009260 comp rx=0 tx=0).stop 2023-07-14T11:09:26.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 msgr2=0x7f29cc19f000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:26.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f29c400c370 tx=0x7f29c400c680 comp rx=0 tx=0).stop 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 shutdown_connections 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f29bc0a1220 0x7f29bc0a36d0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f29cc101680 0x7f29cc19f000 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f29cc0ffb00 0x7f29cc197a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 --2- 172.21.15.40:0/3496452987 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f29cc100cf0 0x7f29cc197f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 >> 172.21.15.40:0/3496452987 conn(0x7f29cc0fb0e0 msgr2=0x7f29cc106580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:26.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.495+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 shutdown_connections 2023-07-14T11:09:26.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:26.496+0000 7f29d2dff700 1 -- 172.21.15.40:0/3496452987 wait complete. 2023-07-14T11:09:26.500 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 8 2023-07-14T11:09:27.006 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:26 smithi045 systemd[1]: Starting Ceph mds.h for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:27.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:27 smithi130 ceph-mon[129405]: Deploying daemon mds.h on smithi045 2023-07-14T11:09:27.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:boot 2023-07-14T11:09:27.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:27 smithi130 ceph-mon[129405]: fsmap 7 up:standby 2023-07-14T11:09:27.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:27 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:09:27.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:27 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3496452987' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:27.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:27 smithi040 ceph-mon[97954]: Deploying daemon mds.h on smithi045 2023-07-14T11:09:27.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:boot 2023-07-14T11:09:27.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:27 smithi040 ceph-mon[97954]: fsmap 7 up:standby 2023-07-14T11:09:27.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:27 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:09:27.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:27 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3496452987' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:27.299 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mon[128791]: Deploying daemon mds.h on smithi045 2023-07-14T11:09:27.299 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:boot 2023-07-14T11:09:27.299 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mon[128791]: fsmap 7 up:standby 2023-07-14T11:09:27.300 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:09:27.300 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3496452987' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:27.300 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:27 smithi045 podman[153798]: 2023-07-14T11:09:27.564 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:27 smithi045 bash[153798]: e5845f21b10743ac57a2175931c4e0cd093ce643ad2f1dae2d158fcaf1e4362f 2023-07-14T11:09:27.564 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-mds[153838]: -- 172.21.15.45:0/1635743153 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c6333df1e0 con 0x55c6333b4c00 2023-07-14T11:09:27.564 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: starting mds.h at 2023-07-14T11:09:27.565 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:09:27 smithi045 systemd[1]: Started Ceph mds.h for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:28 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:28 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.i", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:28.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:28 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:28.517 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:28.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.729+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/592390451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64074550 msgr2=0x7fbf640729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.729+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/592390451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64074550 0x7fbf640729a0 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fbf54009990 tx=0x7fbf54009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/592390451 shutdown_connections 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/592390451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf64109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/592390451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64074550 0x7fbf640729a0 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/592390451 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64072f70 0x7fbf640733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/592390451 >> 172.21.15.40:0/592390451 conn(0x7fbf640fb0e0 msgr2=0x7fbf640fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:28.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/592390451 shutdown_connections 2023-07-14T11:09:28.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.730+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/592390451 wait complete. 2023-07-14T11:09:28.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 Processor -- start 2023-07-14T11:09:28.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 -- start start 2023-07-14T11:09:28.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64072f70 0x7fbf64197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:28.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64074550 0x7fbf64197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf6419f5f0 con 0x7fbf64072f70 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf6419f730 con 0x7fbf64074550 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf6a7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf6419f8c0 con 0x7fbf641079c0 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf68d9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf63fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64072f70 0x7fbf64197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:28.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf68d9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:56718/0 (socket says 172.21.15.40:56718) 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf68d9c700 1 -- 172.21.15.40:0/1702241891 learned_addr learned my addr 172.21.15.40:0/1702241891 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.731+0000 7fbf637fe700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64074550 0x7fbf64197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 -- 172.21.15.40:0/1702241891 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64074550 msgr2=0x7fbf64197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64074550 0x7fbf64197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 -- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64072f70 msgr2=0x7fbf64197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64072f70 0x7fbf64197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 -- 172.21.15.40:0/1702241891 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf540095f0 con 0x7fbf641079c0 2023-07-14T11:09:28.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf68d9c700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fbf5c007ee0 tx=0x7fbf5c00d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:28.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf5c017070 con 0x7fbf641079c0 2023-07-14T11:09:28.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf6419fba0 con 0x7fbf641079c0 2023-07-14T11:09:28.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbf5c00f040 con 0x7fbf641079c0 2023-07-14T11:09:28.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbf5c005550 con 0x7fbf641079c0 2023-07-14T11:09:28.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.732+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf641a00b0 con 0x7fbf641079c0 2023-07-14T11:09:28.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.733+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf64106070 con 0x7fbf641079c0 2023-07-14T11:09:28.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.735+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fbf5c00db20 con 0x7fbf641079c0 2023-07-14T11:09:28.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.737+0000 7fbf617fa700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 0x7fbf4c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:28.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.737+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7fbf5c0cc240 con 0x7fbf641079c0 2023-07-14T11:09:28.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.737+0000 7fbf63fff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 0x7fbf4c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:28.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.738+0000 7fbf63fff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 0x7fbf4c0a3650 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fbf5400b2e0 tx=0x7fbf54005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:28.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.739+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fbf5c095d90 con 0x7fbf641079c0 2023-07-14T11:09:28.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.993+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fbf6402c850 con 0x7fbf641079c0 2023-07-14T11:09:28.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.994+0000 7fbf617fa700 1 -- 172.21.15.40:0/1702241891 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 9 v9) v1 ==== 93+0+6562 (secure 0 0 0) 0x7fbf5c0993b0 con 0x7fbf641079c0 2023-07-14T11:09:28.997 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:28.997 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":9,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8}],"filesystems":[]} 2023-07-14T11:09:28.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 msgr2=0x7fbf4c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:28.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 0x7fbf4c0a3650 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7fbf5400b2e0 tx=0x7fbf54005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:28.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 msgr2=0x7fbf6419f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7fbf5c007ee0 tx=0x7fbf5c00d3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 shutdown_connections 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbf4c0a11a0 0x7fbf4c0a3650 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbf641079c0 0x7fbf6419f020 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf64072f70 0x7fbf64197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 --2- 172.21.15.40:0/1702241891 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbf64074550 0x7fbf64197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.996+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 >> 172.21.15.40:0/1702241891 conn(0x7fbf640fb0e0 msgr2=0x7fbf640fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:29.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.997+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 shutdown_connections 2023-07-14T11:09:29.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:28.997+0000 7fbf6a7ff700 1 -- 172.21.15.40:0/1702241891 wait complete. 2023-07-14T11:09:29.001 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 9 2023-07-14T11:09:29.293 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 systemd[1]: Starting Ceph mds.i for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:29.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mon[129405]: Deploying daemon mds.i on smithi130 2023-07-14T11:09:29.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:boot 2023-07-14T11:09:29.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mon[129405]: fsmap 8 up:standby 2023-07-14T11:09:29.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:09:29.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1702241891' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:29.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:29 smithi040 ceph-mon[97954]: Deploying daemon mds.i on smithi130 2023-07-14T11:09:29.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:29 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:boot 2023-07-14T11:09:29.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:29 smithi040 ceph-mon[97954]: fsmap 8 up:standby 2023-07-14T11:09:29.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:29 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:09:29.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:29 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1702241891' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:29.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:29 smithi045 ceph-mon[128791]: Deploying daemon mds.i on smithi130 2023-07-14T11:09:29.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:29 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:boot 2023-07-14T11:09:29.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:29 smithi045 ceph-mon[128791]: fsmap 8 up:standby 2023-07-14T11:09:29.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:29 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:09:29.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:29 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1702241891' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:29.660 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 podman[155526]: 2023-07-14T11:09:30.025 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-mds[155567]: -- 172.21.15.130:0/1602869 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56082b1f71e0 con 0x56082b1ccc00 2023-07-14T11:09:30.025 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: starting mds.i at 2023-07-14T11:09:30.025 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 bash[155526]: 83a48a6123ce99def37e34fe18e065f3e58d341b8f47f3293b87a1a54f82f0b1 2023-07-14T11:09:30.025 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:09:29 smithi130 systemd[1]: Started Ceph mds.i for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:30.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:30.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:30 smithi130 ceph-mon[129405]: Deploying daemon mds.g on smithi040 2023-07-14T11:09:30.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:30 smithi040 ceph-mon[97954]: Deploying daemon mds.g on smithi040 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: pgmap v261: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.g", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:30.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:30 smithi045 ceph-mon[128791]: Deploying daemon mds.g on smithi040 2023-07-14T11:09:31.019 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 -- 172.21.15.40:0/2509555195 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f98740ffb00 msgr2=0x7f98740fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 --2- 172.21.15.40:0/2509555195 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f98740ffb00 0x7f98740fff10 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f9870009990 tx=0x7f9870009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 -- 172.21.15.40:0/2509555195 shutdown_connections 2023-07-14T11:09:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 --2- 172.21.15.40:0/2509555195 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874101680 0x7f9874107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 --2- 172.21.15.40:0/2509555195 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f98740ffb00 0x7f98740fff10 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 --2- 172.21.15.40:0/2509555195 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9874100cf0 0x7f9874101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 -- 172.21.15.40:0/2509555195 >> 172.21.15.40:0/2509555195 conn(0x7f98740fb0e0 msgr2=0x7f98740fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:31.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 -- 172.21.15.40:0/2509555195 shutdown_connections 2023-07-14T11:09:31.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.229+0000 7f987b3ff700 1 -- 172.21.15.40:0/2509555195 wait complete. 2023-07-14T11:09:31.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 Processor -- start 2023-07-14T11:09:31.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 -- start start 2023-07-14T11:09:31.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:31.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 0x7f9874197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:31.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9874101680 0x7f987419f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f987419f5e0 con 0x7f98740ffb00 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f987419f720 con 0x7f9874101680 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.231+0000 7f987b3ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f987419f8b0 con 0x7f9874100cf0 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987919b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 0x7f9874197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:31.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987919b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:44612/0 (socket says 172.21.15.40:44612) 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987919b700 1 -- 172.21.15.40:0/3812225680 learned_addr learned my addr 172.21.15.40:0/3812225680 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987999c700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9874101680 0x7f987419f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 -- 172.21.15.40:0/3812225680 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9874101680 msgr2=0x7f987419f010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9874101680 0x7f987419f010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 -- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 msgr2=0x7f9874197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 -- 172.21.15.40:0/3812225680 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9864009520 con 0x7f9874100cf0 2023-07-14T11:09:31.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987919b700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:31.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.232+0000 7f987899a700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 0x7f9874197fa0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f986400e9f0 tx=0x7f986400ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.233+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f986400cb70 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.233+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f986400ee90 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.233+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98700095f0 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.233+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9864003cf0 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.233+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f987419ff10 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.234+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f98741921e0 con 0x7f9874100cf0 2023-07-14T11:09:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.236+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f986400ccd0 con 0x7f9874100cf0 2023-07-14T11:09:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.238+0000 7f986a7fc700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 0x7f98600a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.238+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f9864014070 con 0x7f9874100cf0 2023-07-14T11:09:31.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.238+0000 7f987919b700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 0x7f98600a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:31.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.238+0000 7f987919b700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 0x7f98600a3650 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f9870005b20 tx=0x7f9870005a90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:31.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.240+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9864096460 con 0x7f9874100cf0 2023-07-14T11:09:31.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.529+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f987402c850 con 0x7f9874100cf0 2023-07-14T11:09:31.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.531+0000 7f986a7fc700 1 -- 172.21.15.40:0/3812225680 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 10 v10) v1 ==== 94+0+7324 (secure 0 0 0) 0x7f9864099a80 con 0x7f9874100cf0 2023-07-14T11:09:31.533 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:31.534 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":10,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10}],"filesystems":[]} 2023-07-14T11:09:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 msgr2=0x7f98600a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 0x7f98600a3650 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f9870005b20 tx=0x7f9870005a90 comp rx=0 tx=0).stop 2023-07-14T11:09:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 msgr2=0x7f9874197fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 0x7f9874197fa0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f986400e9f0 tx=0x7f986400ed00 comp rx=0 tx=0).stop 2023-07-14T11:09:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 shutdown_connections 2023-07-14T11:09:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f98600a11a0 0x7f98600a3650 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9874101680 0x7f987419f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98740ffb00 0x7f9874197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 --2- 172.21.15.40:0/3812225680 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9874100cf0 0x7f9874197fa0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.533+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 >> 172.21.15.40:0/3812225680 conn(0x7f98740fb0e0 msgr2=0x7f98741065d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.534+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 shutdown_connections 2023-07-14T11:09:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:31.534+0000 7f987b3ff700 1 -- 172.21.15.40:0/3812225680 wait complete. 2023-07-14T11:09:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 10 2023-07-14T11:09:31.765 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6838/3942717247,v1:172.21.15.130:6839/3942717247] up:boot 2023-07-14T11:09:31.765 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:31 smithi040 ceph-mon[97954]: fsmap 9 up:standby 2023-07-14T11:09:31.765 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:31 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:09:31.766 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:31 smithi040 ceph-mon[97954]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6838/3942717247,v1:172.21.15.130:6839/3942717247] up:boot 2023-07-14T11:09:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:31 smithi130 ceph-mon[129405]: fsmap 9 up:standby 2023-07-14T11:09:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:31 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:09:31.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:31 smithi130 ceph-mon[129405]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:31.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6838/3942717247,v1:172.21.15.130:6839/3942717247] up:boot 2023-07-14T11:09:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:31 smithi045 ceph-mon[128791]: fsmap 9 up:standby 2023-07-14T11:09:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:31 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:09:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:31 smithi045 ceph-mon[128791]: pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:32.296 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 systemd[1]: Starting Ceph mds.g for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:32.651 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 podman[131777]: 2023-07-14T11:09:32.651 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:32 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3812225680' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:32.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:32 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3812225680' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:32.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:32 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3812225680' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:33.045 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 bash[131777]: 2260d8b4dc34caeb9cf146b49343f2c4a6fb72da609b69cdb1743e2321c9d0f2 2023-07-14T11:09:33.046 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 ceph-mds[131817]: -- 172.21.15.40:0/1010648144 <== mon.0 v2:172.21.15.40:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55876efbb1e0 con 0x55876ef90800 2023-07-14T11:09:33.046 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: starting mds.g at 2023-07-14T11:09:33.046 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:09:32 smithi040 systemd[1]: Started Ceph mds.g for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:33.555 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:33.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/2771198450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f540744d0 msgr2=0x7f9f54072920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:33.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/2771198450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f540744d0 0x7f9f54072920 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f9f50009990 tx=0x7f9f50009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/2771198450 shutdown_connections 2023-07-14T11:09:33.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/2771198450 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f541079c0 0x7f9f54109e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/2771198450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f540744d0 0x7f9f54072920 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/2771198450 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f54072ef0 0x7f9f54073360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/2771198450 >> 172.21.15.40:0/2771198450 conn(0x7f9f540fb110 msgr2=0x7f9f540fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:33.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/2771198450 shutdown_connections 2023-07-14T11:09:33.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.773+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/2771198450 wait complete. 2023-07-14T11:09:33.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 Processor -- start 2023-07-14T11:09:33.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 -- start start 2023-07-14T11:09:33.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:33.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f540744d0 0x7f9f54197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 0x7f9f5419efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f5419f590 con 0x7f9f541079c0 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f5419f6d0 con 0x7f9f540744d0 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5c1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f5419f860 con 0x7f9f54072ef0 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f59f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f59f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:35566/0 (socket says 172.21.15.40:35566) 2023-07-14T11:09:33.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f59f9b700 1 -- 172.21.15.40:0/739908719 learned_addr learned my addr 172.21.15.40:0/739908719 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.775+0000 7f9f5a79c700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 0x7f9f5419efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f5979a700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f540744d0 0x7f9f54197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 -- 172.21.15.40:0/739908719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f540744d0 msgr2=0x7f9f54197f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f540744d0 0x7f9f54197f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 -- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 msgr2=0x7f9f5419efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 0x7f9f5419efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:33.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 -- 172.21.15.40:0/739908719 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f500095f0 con 0x7f9f54072ef0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f5a79c700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 0x7f9f5419efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f59f9b700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f9f50004740 tx=0x7f9f50009580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f5001d070 con 0x7f9f54072ef0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9f50004830 con 0x7f9f54072ef0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f5419fae0 con 0x7f9f54072ef0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.776+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f50005450 con 0x7f9f54072ef0 2023-07-14T11:09:33.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.777+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f5419ff00 con 0x7f9f54072ef0 2023-07-14T11:09:33.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.778+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f54106810 con 0x7f9f54072ef0 2023-07-14T11:09:33.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.780+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9f5000fb00 con 0x7f9f54072ef0 2023-07-14T11:09:33.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.781+0000 7f9f4affd700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 0x7f9f400a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:33.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.781+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f9f500cd630 con 0x7f9f54072ef0 2023-07-14T11:09:33.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.781+0000 7f9f5979a700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 0x7f9f400a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:33.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.782+0000 7f9f5979a700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 0x7f9f400a36d0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f9f44009c60 tx=0x7f9f44009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:33.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:33.783+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9f50097210 con 0x7f9f54072ef0 2023-07-14T11:09:33.903 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:33.904 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:33.905 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: Deploying daemon mds.a on smithi040 2023-07-14T11:09:33.905 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6838/3591632989,v1:172.21.15.40:6839/3591632989] up:boot 2023-07-14T11:09:33.905 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: fsmap 10 up:standby 2023-07-14T11:09:33.905 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:33 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:09:34.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.050+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f9f5402c850 con 0x7f9f54072ef0 2023-07-14T11:09:34.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.051+0000 7f9f4affd700 1 -- 172.21.15.40:0/739908719 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 11 v11) v1 ==== 94+0+8082 (secure 0 0 0) 0x7f9f5009a830 con 0x7f9f54072ef0 2023-07-14T11:09:34.054 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:34.054 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":11,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24553,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11}],"filesystems":[]} 2023-07-14T11:09:34.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 msgr2=0x7f9f400a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:34.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 0x7f9f400a36d0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f9f44009c60 tx=0x7f9f44009260 comp rx=0 tx=0).stop 2023-07-14T11:09:34.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 msgr2=0x7f9f54197a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:34.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f9f50004740 tx=0x7f9f50009580 comp rx=0 tx=0).stop 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 shutdown_connections 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9f400a1220 0x7f9f400a36d0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f541079c0 0x7f9f5419efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f54072ef0 0x7f9f54197a10 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 --2- 172.21.15.40:0/739908719 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f540744d0 0x7f9f54197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 >> 172.21.15.40:0/739908719 conn(0x7f9f540fb110 msgr2=0x7f9f54103950 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:34.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.053+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 shutdown_connections 2023-07-14T11:09:34.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:34.054+0000 7f9f5c1ff700 1 -- 172.21.15.40:0/739908719 wait complete. 2023-07-14T11:09:34.058 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 11 2023-07-14T11:09:34.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: Deploying daemon mds.a on smithi040 2023-07-14T11:09:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6838/3591632989,v1:172.21.15.40:6839/3591632989] up:boot 2023-07-14T11:09:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: fsmap 10 up:standby 2023-07-14T11:09:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:33 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: Deploying daemon mds.a on smithi040 2023-07-14T11:09:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6838/3591632989,v1:172.21.15.40:6839/3591632989] up:boot 2023-07-14T11:09:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: fsmap 10 up:standby 2023-07-14T11:09:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:33 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:09:35.114 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:34 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/739908719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:35.114 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 systemd[1]: Starting Ceph mds.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:35.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:34 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/739908719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:35.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:34 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/739908719' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:35.863 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 podman[132168]: 2023-07-14T11:09:35.863 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 bash[132168]: 29e47f58b83bc70a1ebfd262670ffde13a73ebc383b3d6430e5b5b8bea606939 2023-07-14T11:09:35.863 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 ceph-mds[132208]: -- 172.21.15.40:0/574992806 <== mon.0 v2:172.21.15.40:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5615986b91e0 con 0x56159868ec00 2023-07-14T11:09:35.863 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: starting mds.a at 2023-07-14T11:09:36.077 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:36.142 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:09:35 smithi040 systemd[1]: Started Ceph mds.a for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:36.143 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:35 smithi040 ceph-mon[97954]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:36.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 -- 172.21.15.40:0/3029177875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 msgr2=0x7f1c0c102110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:36.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 --2- 172.21.15.40:0/3029177875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c102110 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f1c08009990 tx=0x7f1c08009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 -- 172.21.15.40:0/3029177875 shutdown_connections 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 --2- 172.21.15.40:0/3029177875 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 0x7f1c0c10a140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 --2- 172.21.15.40:0/3029177875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c102110 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 --2- 172.21.15.40:0/3029177875 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c103340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 -- 172.21.15.40:0/3029177875 >> 172.21.15.40:0/3029177875 conn(0x7f1c0c0fd320 msgr2=0x7f1c0c0ff750 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 -- 172.21.15.40:0/3029177875 shutdown_connections 2023-07-14T11:09:36.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.285+0000 7f1c145ff700 1 -- 172.21.15.40:0/3029177875 wait complete. 2023-07-14T11:09:36.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.286+0000 7f1c145ff700 1 Processor -- start 2023-07-14T11:09:36.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.286+0000 7f1c145ff700 1 -- start start 2023-07-14T11:09:36.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.286+0000 7f1c145ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:36.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.286+0000 7f1c145ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.286+0000 7f1c145ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 0x7f1c0c19efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c145ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c0c19f590 con 0x7f1c0c101d00 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c145ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c0c19f6d0 con 0x7f1c0c102ef0 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c145ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c0c19f860 con 0x7f1c0c103880 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:36.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c1239b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48050/0 (socket says 172.21.15.40:48050) 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 -- 172.21.15.40:0/1437499500 learned_addr learned my addr 172.21.15.40:0/1437499500 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c12b9c700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 0x7f1c0c19efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 -- 172.21.15.40:0/1437499500 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 msgr2=0x7f1c0c19efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 0x7f1c0c19efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 -- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 msgr2=0x7f1c0c197a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c197a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 -- 172.21.15.40:0/1437499500 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c080095f0 con 0x7f1c0c102ef0 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c1239b700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.287+0000 7f1c11b9a700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f1bfc00e9a0 tx=0x7f1bfc00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.288+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bfc009790 con 0x7f1c0c102ef0 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.288+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1bfc00cbd0 con 0x7f1c0c102ef0 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.288+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1bfc018680 con 0x7f1c0c102ef0 2023-07-14T11:09:36.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.288+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c0c19fb40 con 0x7f1c0c102ef0 2023-07-14T11:09:36.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.288+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c0c1a0000 con 0x7f1c0c102ef0 2023-07-14T11:09:36.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:35 smithi130 ceph-mon[129405]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:36.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.289+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c0c109240 con 0x7f1c0c102ef0 2023-07-14T11:09:36.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.291+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1bfc00cd40 con 0x7f1c0c102ef0 2023-07-14T11:09:36.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.292+0000 7f1c037fe700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 0x7f1bf80a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:36.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.292+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f1bfc014070 con 0x7f1c0c102ef0 2023-07-14T11:09:36.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.293+0000 7f1c1239b700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 0x7f1bf80a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:36.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.293+0000 7f1c1239b700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 0x7f1bf80a3650 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f1c0800b2e0 tx=0x7f1c08005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:36.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.294+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1bfc096370 con 0x7f1c0c102ef0 2023-07-14T11:09:36.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:35 smithi045 ceph-mon[128791]: pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:36.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.551+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f1c0c02c850 con 0x7f1c0c102ef0 2023-07-14T11:09:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.552+0000 7f1c037fe700 1 -- 172.21.15.40:0/1437499500 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 12 v12) v1 ==== 94+0+8840 (secure 0 0 0) 0x7f1bfc099990 con 0x7f1c0c102ef0 2023-07-14T11:09:36.555 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:36.555 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":12,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24553,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24565,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12}],"filesystems":[]} 2023-07-14T11:09:36.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 msgr2=0x7f1bf80a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:36.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 0x7f1bf80a3650 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f1c0800b2e0 tx=0x7f1c08005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 msgr2=0x7f1c0c197f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f1bfc00e9a0 tx=0x7f1bfc00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 shutdown_connections 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1bf80a11a0 0x7f1bf80a3650 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1c0c103880 0x7f1c0c19efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c0c101d00 0x7f1c0c197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 --2- 172.21.15.40:0/1437499500 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1c0c102ef0 0x7f1c0c197f50 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 >> 172.21.15.40:0/1437499500 conn(0x7f1c0c0fd320 msgr2=0x7f1c0c106380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 shutdown_connections 2023-07-14T11:09:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:36.555+0000 7f1c145ff700 1 -- 172.21.15.40:0/1437499500 wait complete. 2023-07-14T11:09:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 12 2023-07-14T11:09:37.265 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:37.266 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: Deploying daemon mds.b on smithi045 2023-07-14T11:09:37.267 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:boot 2023-07-14T11:09:37.267 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: fsmap 11 up:standby 2023-07-14T11:09:37.267 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:09:37.267 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:36 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1437499500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:37.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: Deploying daemon mds.b on smithi045 2023-07-14T11:09:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:boot 2023-07-14T11:09:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: fsmap 11 up:standby 2023-07-14T11:09:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:09:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:36 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1437499500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:37.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-07-14T11:09:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-07-14T11:09:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: Deploying daemon mds.b on smithi045 2023-07-14T11:09:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:boot 2023-07-14T11:09:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: fsmap 11 up:standby 2023-07-14T11:09:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:09:37.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:36 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1437499500' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:37.814 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:37 smithi045 systemd[1]: Starting Ceph mds.b for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:09:38.079 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:37 smithi045 ceph-mon[128791]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:38.079 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:37 smithi045 podman[154141]: 2023-07-14T11:09:38.079 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:38 smithi045 bash[154141]: 7ed87b201a087a65d79a9aff1cf7723b77c7fe1252987d62a45471da46b18896 2023-07-14T11:09:38.080 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:38 smithi045 ceph-mds[154181]: -- 172.21.15.45:0/422321640 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555b7d75b1e0 con 0x555b7d731000 2023-07-14T11:09:38.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:37 smithi130 ceph-mon[129405]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:38.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:37 smithi040 ceph-mon[97954]: pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:38.563 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: starting mds.b at 2023-07-14T11:09:38.564 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:38 smithi045 systemd[1]: Started Ceph mds.b for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:09:38.576 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:38.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.781+0000 7f409e79d700 1 -- 172.21.15.40:0/142097957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098105760 msgr2=0x7f4098107b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:38.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.781+0000 7f409e79d700 1 --2- 172.21.15.40:0/142097957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098105760 0x7f4098107b40 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f4090009f90 tx=0x7f409000b330 comp rx=0 tx=0).stop 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 -- 172.21.15.40:0/142097957 shutdown_connections 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 --2- 172.21.15.40:0/142097957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098105760 0x7f4098107b40 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 --2- 172.21.15.40:0/142097957 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4098100520 0x7f4098102900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 --2- 172.21.15.40:0/142097957 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f4098105220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 -- 172.21.15.40:0/142097957 >> 172.21.15.40:0/142097957 conn(0x7f40980f9ee0 msgr2=0x7f40980fc330 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 -- 172.21.15.40:0/142097957 shutdown_connections 2023-07-14T11:09:38.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.782+0000 7f409e79d700 1 -- 172.21.15.40:0/142097957 wait complete. 2023-07-14T11:09:38.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.783+0000 7f409e79d700 1 Processor -- start 2023-07-14T11:09:38.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.783+0000 7f409e79d700 1 -- start start 2023-07-14T11:09:38.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:38.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:38.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4098105760 0x7f40980fea10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:38.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40980ff1b0 con 0x7f4098100520 2023-07-14T11:09:38.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40980ff320 con 0x7f4098105760 2023-07-14T11:09:38.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409e79d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f409819f670 con 0x7f4098102e40 2023-07-14T11:09:38.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409d79b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:38.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:38.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:35634/0 (socket says 172.21.15.40:35634) 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 -- 172.21.15.40:0/2251870784 learned_addr learned my addr 172.21.15.40:0/2251870784 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409d79b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58272/0 (socket says 172.21.15.40:58272) 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 -- 172.21.15.40:0/2251870784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4098105760 msgr2=0x7f40980fea10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4098105760 0x7f40980fea10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 -- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 msgr2=0x7f40980ffe90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409cf9a700 1 -- 172.21.15.40:0/2251870784 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4088009520 con 0x7f4098102e40 2023-07-14T11:09:38.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.784+0000 7f409d79b700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f409cf9a700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f408800eb30 tx=0x7f408800eef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4088009880 con 0x7f4098102e40 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4090009c40 con 0x7f4098102e40 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f40880044e0 con 0x7f4098102e40 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4088003df0 con 0x7f4098102e40 2023-07-14T11:09:38.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.785+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f409819fd20 con 0x7f4098102e40 2023-07-14T11:09:38.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.786+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f40980fba50 con 0x7f4098102e40 2023-07-14T11:09:38.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.789+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4088004000 con 0x7f4098102e40 2023-07-14T11:09:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.791+0000 7f408e7fc700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 0x7f40840a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.791+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f4088014070 con 0x7f4098102e40 2023-07-14T11:09:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.791+0000 7f409d79b700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 0x7f40840a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.792+0000 7f409d79b700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 0x7f40840a3770 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4094005fd0 tx=0x7f4094009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:38.795+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4088096750 con 0x7f4098102e40 2023-07-14T11:09:39.065 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.063+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f409802c850 con 0x7f4098102e40 2023-07-14T11:09:39.066 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.064+0000 7f408e7fc700 1 -- 172.21.15.40:0/2251870784 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 13 v13) v1 ==== 94+0+9595 (secure 0 0 0) 0x7f4088099d70 con 0x7f4098102e40 2023-07-14T11:09:39.067 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:39.067 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":13,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24553,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24565,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12},{"gid":24569,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":13}],"filesystems":[]} 2023-07-14T11:09:39.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 msgr2=0x7f40840a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 0x7f40840a3770 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f4094005fd0 tx=0x7f4094009310 comp rx=0 tx=0).stop 2023-07-14T11:09:39.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 msgr2=0x7f40980fe4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f408800eb30 tx=0x7f408800eef0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 shutdown_connections 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.066+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f40840a12c0 0x7f40840a3770 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4098105760 0x7f40980fea10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4098100520 0x7f40980ffe90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 --2- 172.21.15.40:0/2251870784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4098102e40 0x7f40980fe4d0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 >> 172.21.15.40:0/2251870784 conn(0x7f40980f9ee0 msgr2=0x7f4098101180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:39.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 shutdown_connections 2023-07-14T11:09:39.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.067+0000 7f409e79d700 1 -- 172.21.15.40:0/2251870784 wait complete. 2023-07-14T11:09:39.071 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 13 2023-07-14T11:09:39.089 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:39.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 -- 172.21.15.40:0/1110983588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce00ffb00 msgr2=0x7f6ce00fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/1110983588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce00ffb00 0x7f6ce00fff10 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00faf0 tx=0x7f6cdc00fe00 comp rx=0 tx=0).stop 2023-07-14T11:09:39.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 -- 172.21.15.40:0/1110983588 shutdown_connections 2023-07-14T11:09:39.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/1110983588 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0101680 0x7f6ce0107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/1110983588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce00ffb00 0x7f6ce00fff10 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/1110983588 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce0100cf0 0x7f6ce0101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 -- 172.21.15.40:0/1110983588 >> 172.21.15.40:0/1110983588 conn(0x7f6ce00fb0e0 msgr2=0x7f6ce00fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:39.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 -- 172.21.15.40:0/1110983588 shutdown_connections 2023-07-14T11:09:39.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.302+0000 7f6ce85ff700 1 -- 172.21.15.40:0/1110983588 wait complete. 2023-07-14T11:09:39.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.303+0000 7f6ce85ff700 1 Processor -- start 2023-07-14T11:09:39.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 -- start start 2023-07-14T11:09:39.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0100cf0 0x7f6ce0193bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce0101680 0x7f6ce019ac20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce019b1f0 con 0x7f6ce0101680 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce019b330 con 0x7f6ce0100cf0 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce85ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6ce019b4c0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce639b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce639b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:35648/0 (socket says 172.21.15.40:35648) 2023-07-14T11:09:39.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce639b700 1 -- 172.21.15.40:0/2498996985 learned_addr learned my addr 172.21.15.40:0/2498996985 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:39.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.304+0000 7f6ce6b9c700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce0101680 0x7f6ce019ac20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce5b9a700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0100cf0 0x7f6ce0193bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 -- 172.21.15.40:0/2498996985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0100cf0 msgr2=0x7f6ce0193bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0100cf0 0x7f6ce0193bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 -- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce0101680 msgr2=0x7f6ce019ac20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce0101680 0x7f6ce019ac20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 -- 172.21.15.40:0/2498996985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6cd8009c40 con 0x7f6ce00ffb00 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.305+0000 7f6ce639b700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00fa40 tx=0x7f6cdc009d80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:39.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.306+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc003fb0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.306+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6cdc00f370 con 0x7f6ce00ffb00 2023-07-14T11:09:39.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.306+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6cdc00f750 con 0x7f6ce00ffb00 2023-07-14T11:09:39.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.306+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6cdc0216c0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.306+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6ce019ba30 con 0x7f6ce00ffb00 2023-07-14T11:09:39.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.307+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6ce0107090 con 0x7f6ce00ffb00 2023-07-14T11:09:39.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.309+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6cdc0041a0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.311+0000 7f6cd77fe700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 0x7f6ccc0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.311+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f6cdc0d1cf0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.311+0000 7f6ce5b9a700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 0x7f6ccc0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.312+0000 7f6ce5b9a700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 0x7f6ccc0a3650 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f6cd0009bf0 tx=0x7f6cd0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:39.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.312+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6cdc09b8d0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.533+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6ce002c850 con 0x7f6ce00ffb00 2023-07-14T11:09:39.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.535+0000 7f6cd77fe700 1 -- 172.21.15.40:0/2498996985 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v80) v1 ==== 74+0+18361 (secure 0 0 0) 0x7f6cdc00eeb0 con 0x7f6ce00ffb00 2023-07-14T11:09:39.538 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:39.538 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":80,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:08:56.533044+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":26,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":75,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 msgr2=0x7f6ccc0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 0x7f6ccc0a3650 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f6cd0009bf0 tx=0x7f6cd0009260 comp rx=0 tx=0).stop 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 msgr2=0x7f6ce0193670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f6cdc00fa40 tx=0x7f6cdc009d80 comp rx=0 tx=0).stop 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 shutdown_connections 2023-07-14T11:09:39.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6ccc0a11a0 0x7f6ccc0a3650 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6ce0101680 0x7f6ce019ac20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6ce00ffb00 0x7f6ce0193670 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 --2- 172.21.15.40:0/2498996985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6ce0100cf0 0x7f6ce0193bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 >> 172.21.15.40:0/2498996985 conn(0x7f6ce00fb0e0 msgr2=0x7f6ce01041d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 shutdown_connections 2023-07-14T11:09:39.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.538+0000 7f6ce85ff700 1 -- 172.21.15.40:0/2498996985 wait complete. 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:39.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:boot 2023-07-14T11:09:39.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: fsmap 12 up:standby 2023-07-14T11:09:39.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:09:39.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:39 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2251870784' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:boot 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: fsmap 12 up:standby 2023-07-14T11:09:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:09:39.547 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:39 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2251870784' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:boot 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: fsmap 12 up:standby 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:09:39.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:39 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2251870784' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:39.567 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2023-07-14T11:09:39.567 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:09:39.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 -- 172.21.15.40:0/3375762757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 msgr2=0x7f0428100020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 --2- 172.21.15.40:0/3375762757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428100020 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f0418009990 tx=0x7f0418009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 -- 172.21.15.40:0/3375762757 shutdown_connections 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 --2- 172.21.15.40:0/3375762757 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f0428106e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 --2- 172.21.15.40:0/3375762757 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 0x7f04280fedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 --2- 172.21.15.40:0/3375762757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428100020 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.792+0000 7f042fdff700 1 -- 172.21.15.40:0/3375762757 >> 172.21.15.40:0/3375762757 conn(0x7f04280fa000 msgr2=0x7f04280fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.793+0000 7f042fdff700 1 -- 172.21.15.40:0/3375762757 shutdown_connections 2023-07-14T11:09:39.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.793+0000 7f042fdff700 1 -- 172.21.15.40:0/3375762757 wait complete. 2023-07-14T11:09:39.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.793+0000 7f042fdff700 1 Processor -- start 2023-07-14T11:09:39.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 -- start start 2023-07-14T11:09:39.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 0x7f0428197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f042819f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f042819f630 con 0x7f0428100560 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f042819f770 con 0x7f04280ffbd0 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042fdff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f042819f900 con 0x7f04280fe9e0 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042e39c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f042819f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48106/0 (socket says 172.21.15.40:48106) 2023-07-14T11:09:39.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042d39a700 1 -- 172.21.15.40:0/313034438 learned_addr learned my addr 172.21.15.40:0/313034438 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.794+0000 7f042db9b700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 0x7f0428197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 -- 172.21.15.40:0/313034438 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 msgr2=0x7f0428197ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 0x7f0428197ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 -- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 msgr2=0x7f042819f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f042819f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 -- 172.21.15.40:0/313034438 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0424009520 con 0x7f04280ffbd0 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042e39c700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f042819f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:39.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042d39a700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f041800b620 tx=0x7f0418005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:39.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f041801d070 con 0x7f04280ffbd0 2023-07-14T11:09:39.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0418022470 con 0x7f04280ffbd0 2023-07-14T11:09:39.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0420009c40 con 0x7f04280ffbd0 2023-07-14T11:09:39.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0418017650 con 0x7f04280ffbd0 2023-07-14T11:09:39.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.795+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04180095f0 con 0x7f04280ffbd0 2023-07-14T11:09:39.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.796+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f042810a200 con 0x7f04280ffbd0 2023-07-14T11:09:39.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.798+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f04180225e0 con 0x7f04280ffbd0 2023-07-14T11:09:39.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.800+0000 7f041effd700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 0x7f04140a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:39.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.800+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f04180cdb00 con 0x7f04280ffbd0 2023-07-14T11:09:39.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.800+0000 7f042db9b700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 0x7f04140a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:39.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.801+0000 7f042db9b700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 0x7f04140a3600 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f04280ffa30 tx=0x7f0424009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:39.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:39.803+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f04180976e0 con 0x7f04280ffbd0 2023-07-14T11:09:40.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.028+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f04281a05f0 con 0x7f04280ffbd0 2023-07-14T11:09:40.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.029+0000 7f041effd700 1 -- 172.21.15.40:0/313034438 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v80) v1 ==== 93+0+10 (secure 0 0 0) 0x7f041809ad00 con 0x7f04280ffbd0 2023-07-14T11:09:40.032 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:09:40.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 msgr2=0x7f04140a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 0x7f04140a3600 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f04280ffa30 tx=0x7f0424009260 comp rx=0 tx=0).stop 2023-07-14T11:09:40.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 msgr2=0x7f0428197ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f041800b620 tx=0x7f0418005740 comp rx=0 tx=0).stop 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 shutdown_connections 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04140a1150 0x7f04140a3600 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0428100560 0x7f042819f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04280fe9e0 0x7f0428197ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 --2- 172.21.15.40:0/313034438 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04280ffbd0 0x7f0428197ff0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 >> 172.21.15.40:0/313034438 conn(0x7f04280fa000 msgr2=0x7f04280676f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.031+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 shutdown_connections 2023-07-14T11:09:40.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.032+0000 7f042fdff700 1 -- 172.21.15.40:0/313034438 wait complete. 2023-07-14T11:09:40.054 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --format=json-pretty osd lspools 2023-07-14T11:09:40.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.264+0000 7faf601ff700 1 -- 172.21.15.40:0/1761041811 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf581016a0 msgr2=0x7faf58107f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.264+0000 7faf601ff700 1 --2- 172.21.15.40:0/1761041811 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf581016a0 0x7faf58107f60 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7faf5000b0a0 tx=0x7faf5000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 -- 172.21.15.40:0/1761041811 shutdown_connections 2023-07-14T11:09:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 --2- 172.21.15.40:0/1761041811 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf581016a0 0x7faf58107f60 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 --2- 172.21.15.40:0/1761041811 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580fff30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 --2- 172.21.15.40:0/1761041811 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf58100d10 0x7faf58101160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 -- 172.21.15.40:0/1761041811 >> 172.21.15.40:0/1761041811 conn(0x7faf580fb160 msgr2=0x7faf580fd570 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 -- 172.21.15.40:0/1761041811 shutdown_connections 2023-07-14T11:09:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.265+0000 7faf601ff700 1 -- 172.21.15.40:0/1761041811 wait complete. 2023-07-14T11:09:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 Processor -- start 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 -- start start 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 0x7faf58072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf581016a0 0x7faf58072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf580735e0 con 0x7faf58100d10 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf58073750 con 0x7faf580ffb20 2023-07-14T11:09:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.266+0000 7faf601ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf5819f9d0 con 0x7faf581016a0 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5d79a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 0x7faf58072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5d79a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 0x7faf58072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58342/0 (socket says 172.21.15.40:58342) 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5d79a700 1 -- 172.21.15.40:0/379483180 learned_addr learned my addr 172.21.15.40:0/379483180 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5e79c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf581016a0 0x7faf58072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 -- 172.21.15.40:0/379483180 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf581016a0 msgr2=0x7faf58072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf581016a0 0x7faf58072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 -- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 msgr2=0x7faf58072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 0x7faf58072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 -- 172.21.15.40:0/379483180 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf50009d00 con 0x7faf580ffb20 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf5df9b700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580742c0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7faf5400d720 tx=0x7faf5400da30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.267+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf5400f690 con 0x7faf580ffb20 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.268+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faf5400fcb0 con 0x7faf580ffb20 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.268+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faf5819fcb0 con 0x7faf580ffb20 2023-07-14T11:09:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.268+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf54010e60 con 0x7faf580ffb20 2023-07-14T11:09:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.268+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf581a0190 con 0x7faf580ffb20 2023-07-14T11:09:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.269+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faf5810b350 con 0x7faf580ffb20 2023-07-14T11:09:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.271+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7faf5400f7f0 con 0x7faf580ffb20 2023-07-14T11:09:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.272+0000 7faf4effd700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 0x7faf440a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.272+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7faf540cc110 con 0x7faf580ffb20 2023-07-14T11:09:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.273+0000 7faf5d79a700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 0x7faf440a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.273+0000 7faf5d79a700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 0x7faf440a3650 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7faf48005950 tx=0x7faf4800a400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.274+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7faf54095c60 con 0x7faf580ffb20 2023-07-14T11:09:40.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.493+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd lspools", "format": "json-pretty"} v 0) v1 -- 0x7faf5810b510 con 0x7faf580ffb20 2023-07-14T11:09:40.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.493+0000 7faf4effd700 1 -- 172.21.15.40:0/379483180 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd lspools", "format": "json-pretty"}]=0 v80) v1 ==== 84+0+65 (secure 0 0 0) 0x7faf54084090 con 0x7faf580ffb20 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout:[ 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout: "poolnum": 1, 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout: "poolname": ".mgr" 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:09:40.497 INFO:teuthology.orchestra.run.smithi040.stdout:] 2023-07-14T11:09:40.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 msgr2=0x7faf440a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 0x7faf440a3650 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7faf48005950 tx=0x7faf4800a400 comp rx=0 tx=0).stop 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 msgr2=0x7faf580742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580742c0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7faf5400d720 tx=0x7faf5400da30 comp rx=0 tx=0).stop 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 shutdown_connections 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faf440a11a0 0x7faf440a3650 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf581016a0 0x7faf58072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ffb20 0x7faf580742c0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 --2- 172.21.15.40:0/379483180 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf58100d10 0x7faf58072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 >> 172.21.15.40:0/379483180 conn(0x7faf580fb160 msgr2=0x7faf580fd330 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:40.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 shutdown_connections 2023-07-14T11:09:40.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.496+0000 7faf601ff700 1 -- 172.21.15.40:0/379483180 wait complete. 2023-07-14T11:09:40.517 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2023-07-14T11:09:40.517 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_metadata --pg_num_min 64 2023-07-14T11:09:40.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/2888444850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 msgr2=0x7f3eb40733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/2888444850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb40733e0 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f3ea4009990 tx=0x7f3ea4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/2888444850 shutdown_connections 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/2888444850 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb41079a0 0x7f3eb4109e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/2888444850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb4074550 0x7f3eb40729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/2888444850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb40733e0 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/2888444850 >> 172.21.15.40:0/2888444850 conn(0x7f3eb40fb0e0 msgr2=0x7f3eb40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.729+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/2888444850 shutdown_connections 2023-07-14T11:09:40.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.730+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/2888444850 wait complete. 2023-07-14T11:09:40.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 Processor -- start 2023-07-14T11:09:40.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 -- start start 2023-07-14T11:09:40.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 0x7f3eb4198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb41079a0 0x7f3eb419f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb419f630 con 0x7f3eb4072f70 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb419f770 con 0x7f3eb4074550 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3ebb7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eb419f900 con 0x7f3eb41079a0 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3eb959b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3eb959b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58348/0 (socket says 172.21.15.40:58348) 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3eb959b700 1 -- 172.21.15.40:0/4198983561 learned_addr learned my addr 172.21.15.40:0/4198983561 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:40.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3eb9d9c700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb41079a0 0x7f3eb419f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.731+0000 7f3eb8d9a700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 0x7f3eb4198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 -- 172.21.15.40:0/4198983561 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb41079a0 msgr2=0x7f3eb419f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb41079a0 0x7f3eb419f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 -- 172.21.15.40:0/4198983561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 msgr2=0x7f3eb4198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 0x7f3eb4198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 -- 172.21.15.40:0/4198983561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ea40095f0 con 0x7f3eb4072f70 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb8d9a700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 0x7f3eb4198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:40.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eb959b700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3eb000e970 tx=0x7f3eb000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:40.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eb0009790 con 0x7f3eb4072f70 2023-07-14T11:09:40.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3eb000c7e0 con 0x7f3eb4072f70 2023-07-14T11:09:40.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3eb00054e0 con 0x7f3eb4072f70 2023-07-14T11:09:40.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.732+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3eb419fbe0 con 0x7f3eb4072f70 2023-07-14T11:09:40.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.733+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3eb41a00a0 con 0x7f3eb4072f70 2023-07-14T11:09:40.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.734+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3eb4106050 con 0x7f3eb4072f70 2023-07-14T11:09:40.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.735+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f3eb000c950 con 0x7f3eb4072f70 2023-07-14T11:09:40.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.737+0000 7f3eaa7fc700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 0x7f3ea00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:40.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.737+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(80..80 src has 1..80) v4 ==== 7127+0+0 (secure 0 0 0) 0x7f3eb0014070 con 0x7f3eb4072f70 2023-07-14T11:09:40.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.737+0000 7f3eb8d9a700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 0x7f3ea00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:40.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.738+0000 7f3eb8d9a700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 0x7f3ea00a3600 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f3ea400b2e0 tx=0x7f3ea4005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.741+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3eb0096280 con 0x7f3eb4072f70 2023-07-14T11:09:40.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:40 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2498996985' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:40.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:40 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/313034438' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:40.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:40 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2498996985' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:40.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:40 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/313034438' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:40.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:40 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2498996985' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:40.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:40 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/313034438' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:09:40.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:40.974+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64} v 0) v1 -- 0x7f3eb402c850 con 0x7f3eb4072f70 2023-07-14T11:09:41.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.560+0000 7f3eaa7fc700 1 -- 172.21.15.40:0/4198983561 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]=0 pool 'cephfs_metadata' created v81) v1 ==== 138+0+0 (secure 0 0 0) 0x7f3eb00998a0 con 0x7f3eb4072f70 2023-07-14T11:09:41.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 msgr2=0x7f3ea00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:41.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 0x7f3ea00a3600 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f3ea400b2e0 tx=0x7f3ea4005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 msgr2=0x7f3eb4197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:41.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f3eb000e970 tx=0x7f3eb000ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 shutdown_connections 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3ea00a1150 0x7f3ea00a3600 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3eb41079a0 0x7f3eb419f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eb4072f70 0x7f3eb4197b00 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 --2- 172.21.15.40:0/4198983561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3eb4074550 0x7f3eb4198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.564+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 >> 172.21.15.40:0/4198983561 conn(0x7f3eb40fb0e0 msgr2=0x7f3eb40fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.565+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 shutdown_connections 2023-07-14T11:09:41.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.565+0000 7f3ebb7ff700 1 -- 172.21.15.40:0/4198983561 wait complete. 2023-07-14T11:09:41.569 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_metadata' created 2023-07-14T11:09:41.585 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.05 2023-07-14T11:09:41.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:41 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/379483180' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-07-14T11:09:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:41 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-07-14T11:09:41.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:41 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/379483180' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-07-14T11:09:41.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:41 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-07-14T11:09:41.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 -- 172.21.15.40:0/376585679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0072f70 msgr2=0x7f7eb00733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:41.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/376585679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0072f70 0x7f7eb00733e0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7ea40098f0 tx=0x7f7ea4009c00 comp rx=0 tx=0).stop 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 -- 172.21.15.40:0/376585679 shutdown_connections 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/376585679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 0x7f7eb0109e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/376585679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0074550 0x7f7eb00729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/376585679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0072f70 0x7f7eb00733e0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 -- 172.21.15.40:0/376585679 >> 172.21.15.40:0/376585679 conn(0x7f7eb00fb130 msgr2=0x7f7eb00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 -- 172.21.15.40:0/376585679 shutdown_connections 2023-07-14T11:09:41.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.802+0000 7f7eb57ff700 1 -- 172.21.15.40:0/376585679 wait complete. 2023-07-14T11:09:41.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 Processor -- start 2023-07-14T11:09:41.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 -- start start 2023-07-14T11:09:41.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 0x7f7eb0197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:41.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:41.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 0x7f7eb019f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb019f640 con 0x7f7eb0072f70 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb019f780 con 0x7f7eb0074550 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eb57ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7eb019f910 con 0x7f7eb01079b0 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eaeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 0x7f7eb0197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eae7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eae7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48140/0 (socket says 172.21.15.40:48140) 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.803+0000 7f7eae7fc700 1 -- 172.21.15.40:0/2106145725 learned_addr learned my addr 172.21.15.40:0/2106145725 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:41.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eaf7fe700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 0x7f7eb019f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 -- 172.21.15.40:0/2106145725 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 msgr2=0x7f7eb019f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 0x7f7eb019f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 -- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 msgr2=0x7f7eb0197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 0x7f7eb0197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 -- 172.21.15.40:0/2106145725 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ea4009520 con 0x7f7eb0074550 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eaeffd700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 0x7f7eb0197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:41.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eae7fc700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4000c00 tx=0x7f7ea400b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:41.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea401c070 con 0x7f7eb0074550 2023-07-14T11:09:41.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7eb019fb90 con 0x7f7eb0074550 2023-07-14T11:09:41.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ea400f7d0 con 0x7f7eb0074550 2023-07-14T11:09:41.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.804+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ea40039b0 con 0x7f7eb0074550 2023-07-14T11:09:41.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.805+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7eb01a0010 con 0x7f7eb0074550 2023-07-14T11:09:41.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.805+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7eb0106060 con 0x7f7eb0074550 2023-07-14T11:09:41.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.808+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f7ea4020030 con 0x7f7eb0074550 2023-07-14T11:09:41.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.809+0000 7f7e97fff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 0x7f7e980a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:41.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.809+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(81..81 src has 1..81) v4 ==== 7497+0+0 (secure 0 0 0) 0x7f7ea40cdf20 con 0x7f7eb0074550 2023-07-14T11:09:41.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.809+0000 7f7eaeffd700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 0x7f7e980a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:41.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.810+0000 7f7eaeffd700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 0x7f7e980a3600 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0074920 tx=0x7f7ea0009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:41.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:41 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/379483180' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2023-07-14T11:09:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:41 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2023-07-14T11:09:41.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:41.811+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f7ea4097990 con 0x7f7eb0074550 2023-07-14T11:09:42.064 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.061+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05} v 0) v1 -- 0x7f7eb002ccd0 con 0x7f7eb0074550 2023-07-14T11:09:42.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.587+0000 7f7e97fff700 1 -- 172.21.15.40:0/2106145725 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]=0 pool 'cephfs_data' created v82) v1 ==== 171+0+0 (secure 0 0 0) 0x7f7eb002ccd0 con 0x7f7eb0074550 2023-07-14T11:09:42.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.597+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 msgr2=0x7f7e980a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:42.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.597+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 0x7f7e980a3600 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0074920 tx=0x7f7ea0009310 comp rx=0 tx=0).stop 2023-07-14T11:09:42.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.597+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 msgr2=0x7f7eb0198000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:42.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.597+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f7ea4000c00 tx=0x7f7ea400b790 comp rx=0 tx=0).stop 2023-07-14T11:09:42.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 shutdown_connections 2023-07-14T11:09:42.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7e980a1150 0x7f7e980a3600 secure :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f7eb0074920 tx=0x7f7ea0009310 comp rx=0 tx=0).stop 2023-07-14T11:09:42.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7eb01079b0 0x7f7eb019f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7eb0072f70 0x7f7eb0197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 --2- 172.21.15.40:0/2106145725 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7eb0074550 0x7f7eb0198000 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.598+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 >> 172.21.15.40:0/2106145725 conn(0x7f7eb00fb130 msgr2=0x7f7eb00fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:42.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.601+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 shutdown_connections 2023-07-14T11:09:42.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.601+0000 7f7eb57ff700 1 -- 172.21.15.40:0/2106145725 wait complete. 2023-07-14T11:09:42.606 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_data' created 2023-07-14T11:09:42.628 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs new cephfs cephfs_metadata cephfs_data 2023-07-14T11:09:42.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 -- 172.21.15.40:0/1800139756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c100cd0 msgr2=0x7fb70c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:42.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1800139756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c100cd0 0x7fb70c101120 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fb6f4009990 tx=0x7fb6f4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 -- 172.21.15.40:0/1800139756 shutdown_connections 2023-07-14T11:09:42.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1800139756 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c101660 0x7fb70c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1800139756 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1800139756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c100cd0 0x7fb70c101120 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 -- 172.21.15.40:0/1800139756 >> 172.21.15.40:0/1800139756 conn(0x7fb70c0fb0e0 msgr2=0x7fb70c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:42.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 -- 172.21.15.40:0/1800139756 shutdown_connections 2023-07-14T11:09:42.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.838+0000 7fb7113ff700 1 -- 172.21.15.40:0/1800139756 wait complete. 2023-07-14T11:09:42.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.839+0000 7fb7113ff700 1 Processor -- start 2023-07-14T11:09:42.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.839+0000 7fb7113ff700 1 -- start start 2023-07-14T11:09:42.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.839+0000 7fb7113ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:42.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb7113ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 0x7fb70c197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:42.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb7113ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 0x7fb70c19efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb7113ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb70c19f5a0 con 0x7fb70c101660 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb7113ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb70c19f6e0 con 0x7fb70c100cd0 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb7113ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb70c19f870 con 0x7fb70c0ffae0 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 0x7fb70c19efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb703fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 0x7fb70c197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:42.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb703fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 0x7fb70c197f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48168/0 (socket says 172.21.15.40:48168) 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb703fff700 1 -- 172.21.15.40:0/1706898535 learned_addr learned my addr 172.21.15.40:0/1706898535 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 -- 172.21.15.40:0/1706898535 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 msgr2=0x7fb70c197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 0x7fb70c197f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 -- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 msgr2=0x7fb70c19efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 0x7fb70c19efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70affd700 1 -- 172.21.15.40:0/1706898535 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6f40095f0 con 0x7fb70c0ffae0 2023-07-14T11:09:42.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.840+0000 7fb70b7fe700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 0x7fb70c19efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:42.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb70affd700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c197a20 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fb6fc00e9c0 tx=0x7fb6fc00ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:42.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6fc004220 con 0x7fb70c0ffae0 2023-07-14T11:09:42.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb6fc004380 con 0x7fb70c0ffae0 2023-07-14T11:09:42.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb70c19fb50 con 0x7fb70c0ffae0 2023-07-14T11:09:42.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb6fc003be0 con 0x7fb70c0ffae0 2023-07-14T11:09:42.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.841+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb70c1a0060 con 0x7fb70c0ffae0 2023-07-14T11:09:42.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.842+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb70c10b2b0 con 0x7fb70c0ffae0 2023-07-14T11:09:42.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.844+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb6fc003d40 con 0x7fb70c0ffae0 2023-07-14T11:09:42.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.846+0000 7fb708ff9700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 0x7fb6f80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:42.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.846+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(82..82 src has 1..82) v4 ==== 7879+0+0 (secure 0 0 0) 0x7fb6fc014070 con 0x7fb70c0ffae0 2023-07-14T11:09:42.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.846+0000 7fb703fff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 0x7fb6f80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:42.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.847+0000 7fb703fff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 0x7fb6f80a3600 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fb6f400b2e0 tx=0x7fb6f4005040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:42.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:42.848+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb6fc0968d0 con 0x7fb70c0ffae0 2023-07-14T11:09:43.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-07-14T11:09:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: osdmap e81: 12 total, 12 up, 12 in 2023-07-14T11:09:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:42 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2106145725' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:43.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:43.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: osdmap e81: 12 total, 12 up, 12 in 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:42 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2106145725' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:43.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4198983561' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: osdmap e81: 12 total, 12 up, 12 in 2023-07-14T11:09:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:43.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:42 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2106145725' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:43.126 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.124+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"} v 0) v1 -- 0x7fb70c10b470 con 0x7fb70c0ffae0 2023-07-14T11:09:43.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.617+0000 7fb708ff9700 1 -- 172.21.15.40:0/1706898535 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]=0 new fs with metadata pool 2 and data pool 3 v14) v1 ==== 172+0+0 (secure 0 0 0) 0x7fb6fc01f340 con 0x7fb70c0ffae0 2023-07-14T11:09:43.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.619+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 msgr2=0x7fb6f80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:43.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.619+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 0x7fb6f80a3600 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fb6f400b2e0 tx=0x7fb6f4005040 comp rx=0 tx=0).stop 2023-07-14T11:09:43.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.619+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 msgr2=0x7fb70c197a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:43.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.619+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c197a20 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7fb6fc00e9c0 tx=0x7fb6fc00ecd0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 shutdown_connections 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb6f80a1150 0x7fb6f80a3600 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb70c101660 0x7fb70c19efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb70c0ffae0 0x7fb70c197a20 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 --2- 172.21.15.40:0/1706898535 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb70c100cd0 0x7fb70c197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.620+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 >> 172.21.15.40:0/1706898535 conn(0x7fb70c0fb0e0 msgr2=0x7fb70c0fd3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:43.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.621+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 shutdown_connections 2023-07-14T11:09:43.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.621+0000 7fb7113ff700 1 -- 172.21.15.40:0/1706898535 wait complete. 2023-07-14T11:09:43.626 INFO:teuthology.orchestra.run.smithi040.stderr:new fs with metadata pool 2 and data pool 3 2023-07-14T11:09:43.645 DEBUG:tasks.cephfs.filesystem:EC profile is ['m=2', 'k=2', 'crush-failure-domain=osd'] 2023-07-14T11:09:43.645 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd erasure-code-profile set cephfs_data_ec m=2 k=2 crush-failure-domain=osd 2023-07-14T11:09:43.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 -- 172.21.15.40:0/1536048352 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 msgr2=0x7f715c0fff00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:43.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 --2- 172.21.15.40:0/1536048352 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0fff00 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f714c009990 tx=0x7f714c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 -- 172.21.15.40:0/1536048352 shutdown_connections 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 --2- 172.21.15.40:0/1536048352 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c101670 0x7f715c107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 --2- 172.21.15.40:0/1536048352 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0fff00 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 --2- 172.21.15.40:0/1536048352 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c100ce0 0x7f715c101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.860+0000 7f7160bff700 1 -- 172.21.15.40:0/1536048352 >> 172.21.15.40:0/1536048352 conn(0x7f715c0fb130 msgr2=0x7f715c0fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.861+0000 7f7160bff700 1 -- 172.21.15.40:0/1536048352 shutdown_connections 2023-07-14T11:09:43.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.861+0000 7f7160bff700 1 -- 172.21.15.40:0/1536048352 wait complete. 2023-07-14T11:09:43.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 Processor -- start 2023-07-14T11:09:43.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 -- start start 2023-07-14T11:09:43.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:43.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 0x7f715c072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c101670 0x7f715c072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f715c0735e0 con 0x7f715c101670 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f715c073750 con 0x7f715c0ffaf0 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7160bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f715c19f860 con 0x7f715c100ce0 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f715a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f715a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39534/0 (socket says 172.21.15.40:39534) 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f715a59c700 1 -- 172.21.15.40:0/829954929 learned_addr learned my addr 172.21.15.40:0/829954929 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:43.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f7159d9b700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 0x7f715c072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.862+0000 7f715ad9d700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c101670 0x7f715c072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 -- 172.21.15.40:0/829954929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 msgr2=0x7f715c072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 0x7f715c072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 -- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c101670 msgr2=0x7f715c072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c101670 0x7f715c072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 -- 172.21.15.40:0/829954929 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f714c0095f0 con 0x7f715c0ffaf0 2023-07-14T11:09:43.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f7159d9b700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 0x7f715c072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f715a59c700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f714c00b620 tx=0x7f714c004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f714c01d070 con 0x7f715c0ffaf0 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f714c005030 con 0x7f715c0ffaf0 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f714c004b80 con 0x7f715c0ffaf0 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f715c19fae0 con 0x7f715c0ffaf0 2023-07-14T11:09:43.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.863+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f715c19feb0 con 0x7f715c0ffaf0 2023-07-14T11:09:43.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.864+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f715c107090 con 0x7f715c0ffaf0 2023-07-14T11:09:43.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.867+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f714c0051a0 con 0x7f715c0ffaf0 2023-07-14T11:09:43.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.868+0000 7f714b7fe700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 0x7f71440a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:43.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.868+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(83..83 src has 1..83) v4 ==== 7979+0+0 (secure 0 0 0) 0x7f714c0cd790 con 0x7f715c0ffaf0 2023-07-14T11:09:43.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.868+0000 7f7159d9b700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 0x7f71440a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:43.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.869+0000 7f7159d9b700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 0x7f71440a3680 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f7150011490 tx=0x7f7150009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:43.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:43.870+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f714c097020 con 0x7f715c0ffaf0 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: osdmap e82: 12 total, 12 up, 12 in 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: pgmap v270: 97 pgs: 11 creating+peering, 1 active+clean, 85 unknown; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1706898535' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:43 smithi130 ceph-mon[129405]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-07-14T11:09:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: osdmap e82: 12 total, 12 up, 12 in 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: pgmap v270: 97 pgs: 11 creating+peering, 1 active+clean, 85 unknown; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1706898535' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:43 smithi040 ceph-mon[97954]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-07-14T11:09:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]: dispatch 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.05}]': finished 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: osdmap e82: 12 total, 12 up, 12 in 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: pgmap v270: 97 pgs: 11 creating+peering, 1 active+clean, 85 unknown; 577 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:09:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:09:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1706898535' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2023-07-14T11:09:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:43 smithi045 ceph-mon[128791]: daemon mds.b assigned to filesystem cephfs as rank 0 2023-07-14T11:09:44.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.101+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]} v 0) v1 -- 0x7f715c02c850 con 0x7f715c0ffaf0 2023-07-14T11:09:44.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.583+0000 7f714b7fe700 1 -- 172.21.15.40:0/829954929 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]=0 v84) v1 ==== 157+0+0 (secure 0 0 0) 0x7f714c004ce0 con 0x7f715c0ffaf0 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 msgr2=0x7f71440a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 0x7f71440a3680 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f7150011490 tx=0x7f7150009260 comp rx=0 tx=0).stop 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 msgr2=0x7f715c0742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f714c00b620 tx=0x7f714c004730 comp rx=0 tx=0).stop 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 shutdown_connections 2023-07-14T11:09:44.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f71440a11d0 0x7f71440a3680 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f715c101670 0x7f715c072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f715c0ffaf0 0x7f715c0742c0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 --2- 172.21.15.40:0/829954929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f715c100ce0 0x7f715c072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 >> 172.21.15.40:0/829954929 conn(0x7f715c0fb130 msgr2=0x7f715c1041d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 shutdown_connections 2023-07-14T11:09:44.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.585+0000 7f7160bff700 1 -- 172.21.15.40:0/829954929 wait complete. 2023-07-14T11:09:44.608 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool create cephfs_data_ec erasure cephfs_data_ec --pg_num_min 64 --target_size_ratio 0.9 2023-07-14T11:09:44.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 -- 172.21.15.40:0/1372879296 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac101690 msgr2=0x7f9dac107f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:44.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1372879296 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac101690 0x7f9dac107f50 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f9da400b0a0 tx=0x7f9da400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 -- 172.21.15.40:0/1372879296 shutdown_connections 2023-07-14T11:09:44.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1372879296 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac101690 0x7f9dac107f50 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1372879296 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac0ffb10 0x7f9dac0fff20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1372879296 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 0x7f9dac101150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 -- 172.21.15.40:0/1372879296 >> 172.21.15.40:0/1372879296 conn(0x7f9dac0fb130 msgr2=0x7f9dac0fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:44.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 -- 172.21.15.40:0/1372879296 shutdown_connections 2023-07-14T11:09:44.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.832+0000 7f9db2dff700 1 -- 172.21.15.40:0/1372879296 wait complete. 2023-07-14T11:09:44.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 Processor -- start 2023-07-14T11:09:44.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 -- start start 2023-07-14T11:09:44.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:44.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 0x7f9dac197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:44.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac101690 0x7f9dac19f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:44.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dac19f5e0 con 0x7f9dac100d00 2023-07-14T11:09:44.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dac19f720 con 0x7f9dac0ffb10 2023-07-14T11:09:44.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.834+0000 7f9db2dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dac19f8b0 con 0x7f9dac101690 2023-07-14T11:09:44.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:44.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39558/0 (socket says 172.21.15.40:39558) 2023-07-14T11:09:44.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 -- 172.21.15.40:0/1073168744 learned_addr learned my addr 172.21.15.40:0/1073168744 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:44.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9dabfff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 0x7f9dac197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:44.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db139c700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac101690 0x7f9dac19f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:44.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 -- 172.21.15.40:0/1073168744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac101690 msgr2=0x7f9dac19f010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:44.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac101690 0x7f9dac19f010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 -- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 msgr2=0x7f9dac197fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:44.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 0x7f9dac197fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:44.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 -- 172.21.15.40:0/1073168744 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9da4009d00 con 0x7f9dac0ffb10 2023-07-14T11:09:44.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.835+0000 7f9db0b9b700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f9da000e970 tx=0x7f9da000ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:44.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.836+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9da0009790 con 0x7f9dac0ffb10 2023-07-14T11:09:44.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.836+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9da000c7e0 con 0x7f9dac0ffb10 2023-07-14T11:09:44.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.836+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9da0003c70 con 0x7f9dac0ffb10 2023-07-14T11:09:44.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.836+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9dac19fb90 con 0x7f9dac0ffb10 2023-07-14T11:09:44.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.836+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dac1a0050 con 0x7f9dac0ffb10 2023-07-14T11:09:44.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.837+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9dac10b340 con 0x7f9dac0ffb10 2023-07-14T11:09:44.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.841+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9da000cc50 con 0x7f9dac0ffb10 2023-07-14T11:09:44.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.843+0000 7f9da9ffb700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 0x7f9d9c0a36a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:44.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.843+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(84..84 src has 1..84) v4 ==== 8205+0+0 (secure 0 0 0) 0x7f9da0014070 con 0x7f9dac0ffb10 2023-07-14T11:09:44.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.843+0000 7f9dabfff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 0x7f9d9c0a36a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:44.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.843+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9da0096800 con 0x7f9dac0ffb10 2023-07-14T11:09:44.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:44.844+0000 7f9dabfff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 0x7f9d9c0a36a0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f9d98009c60 tx=0x7f9d98009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:45.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: osdmap e83: 12 total, 12 up, 12 in 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/829954929' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:09:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:44 smithi130 ceph-mon[129405]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:45.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: osdmap e83: 12 total, 12 up, 12 in 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/829954929' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:09:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:44 smithi040 ceph-mon[97954]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:45.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: osdmap e83: 12 total, 12 up, 12 in 2023-07-14T11:09:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2023-07-14T11:09:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: fsmap cephfs:1 {0=b=up:creating} 11 up:standby 2023-07-14T11:09:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: daemon mds.b is now active in filesystem cephfs as rank 0 2023-07-14T11:09:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/829954929' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]: dispatch 2023-07-14T11:09:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:09:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:44 smithi045 ceph-mon[128791]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:45.066 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:45.061+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9} v 0) v1 -- 0x7f9dac1a0600 con 0x7f9dac0ffb10 2023-07-14T11:09:46.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: osdmap e84: 12 total, 12 up, 12 in 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: pgmap v273: 97 pgs: 19 creating+peering, 27 active+clean, 51 unknown; 578 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail; 767 B/s wr, 2 op/s 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1073168744' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:45 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-07-14T11:09:46.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: osdmap e84: 12 total, 12 up, 12 in 2023-07-14T11:09:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: pgmap v273: 97 pgs: 19 creating+peering, 27 active+clean, 51 unknown; 578 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail; 767 B/s wr, 2 op/s 2023-07-14T11:09:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1073168744' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:45 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "cephfs_data_ec", "profile": ["m=2", "k=2", "crush-failure-domain=osd"]}]': finished 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: osdmap e84: 12 total, 12 up, 12 in 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: pgmap v273: 97 pgs: 19 creating+peering, 27 active+clean, 51 unknown; 578 KiB data, 327 MiB used, 1.0 TiB / 1.0 TiB avail; 767 B/s wr, 2 op/s 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1073168744' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2023-07-14T11:09:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:45 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:46.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.595+0000 7f9da9ffb700 1 -- 172.21.15.40:0/1073168744 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]=0 pool 'cephfs_data_ec' created v86) v1 ==== 228+0+0 (secure 0 0 0) 0x7f9da0003dd0 con 0x7f9dac0ffb10 2023-07-14T11:09:46.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 msgr2=0x7f9d9c0a36a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:46.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 0x7f9d9c0a36a0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f9d98009c60 tx=0x7f9d98009260 comp rx=0 tx=0).stop 2023-07-14T11:09:46.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 msgr2=0x7f9dac197a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:46.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f9da000e970 tx=0x7f9da000ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:46.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 shutdown_connections 2023-07-14T11:09:46.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9d9c0a11f0 0x7f9d9c0a36a0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9dac101690 0x7f9dac19f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9dac0ffb10 0x7f9dac197a60 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 --2- 172.21.15.40:0/1073168744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dac100d00 0x7f9dac197fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.602+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 >> 172.21.15.40:0/1073168744 conn(0x7f9dac0fb130 msgr2=0x7f9dac072740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.603+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 shutdown_connections 2023-07-14T11:09:46.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.603+0000 7f9db2dff700 1 -- 172.21.15.40:0/1073168744 wait complete. 2023-07-14T11:09:46.607 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_data_ec' created 2023-07-14T11:09:46.625 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool set cephfs_data_ec allow_ec_overwrites true 2023-07-14T11:09:46.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2886703089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8100cd0 msgr2=0x7f5ee8101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:46.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2886703089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8101120 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f5ed0009990 tx=0x7f5ed0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2886703089 shutdown_connections 2023-07-14T11:09:46.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2886703089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8101660 0x7f5ee8107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2886703089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 0x7f5ee80ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2886703089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8101120 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2886703089 >> 172.21.15.40:0/2886703089 conn(0x7f5ee80fb0e0 msgr2=0x7f5ee80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:46.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2886703089 shutdown_connections 2023-07-14T11:09:46.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.832+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2886703089 wait complete. 2023-07-14T11:09:46.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.833+0000 7f5eee1ff700 1 Processor -- start 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.833+0000 7f5eee1ff700 1 -- start start 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.833+0000 7f5eee1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 0x7f5ee80742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.833+0000 7f5eee1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.833+0000 7f5eee1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5eee1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ee80735e0 con 0x7f5ee8101660 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5eee1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ee8073750 con 0x7f5ee8100cd0 2023-07-14T11:09:46.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5eee1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ee819f8d0 con 0x7f5ee80ffae0 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5edffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5edffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39584/0 (socket says 172.21.15.40:39584) 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35410/0 (socket says 172.21.15.40:35410) 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5edffff700 1 -- 172.21.15.40:0/2815700770 learned_addr learned my addr 172.21.15.40:0/2815700770 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee77fe700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 0x7f5ee80742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 -- 172.21.15.40:0/2815700770 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 msgr2=0x7f5ee80742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:46.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 0x7f5ee80742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 -- 172.21.15.40:0/2815700770 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 msgr2=0x7f5ee8072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 -- 172.21.15.40:0/2815700770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ed8009520 con 0x7f5ee8101660 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.834+0000 7f5ee7fff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f5ee000efc0 tx=0x7f5ee000c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.835+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ee0018070 con 0x7f5ee8101660 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.835+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ee000f040 con 0x7f5ee8101660 2023-07-14T11:09:46.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.835+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ee000cd40 con 0x7f5ee8101660 2023-07-14T11:09:46.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.835+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ed00095f0 con 0x7f5ee8101660 2023-07-14T11:09:46.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.835+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ee819fed0 con 0x7f5ee8101660 2023-07-14T11:09:46.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.836+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5ee8106a60 con 0x7f5ee8101660 2023-07-14T11:09:46.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.840+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5ee0003a00 con 0x7f5ee8101660 2023-07-14T11:09:46.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.841+0000 7f5ee57fa700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 0x7f5ed40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:46.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.841+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(86..86 src has 1..86) v4 ==== 8698+0+0 (secure 0 0 0) 0x7f5ee00ccff0 con 0x7f5ee8101660 2023-07-14T11:09:46.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.841+0000 7f5ee77fe700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 0x7f5ed40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:46.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.842+0000 7f5ee77fe700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 0x7f5ed40a3680 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f5ee8100b30 tx=0x7f5ed8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:46.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:46.844+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5ee00965c0 con 0x7f5ee8101660 2023-07-14T11:09:47.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: osdmap e85: 12 total, 12 up, 12 in 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:46 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: osdmap e85: 12 total, 12 up, 12 in 2023-07-14T11:09:47.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:47.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:47.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:46 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.053+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"} v 0) v1 -- 0x7f5ee802c850 con 0x7f5ee8101660 2023-07-14T11:09:47.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: osdmap e85: 12 total, 12 up, 12 in 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:09:47.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:46 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:09:47.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.603+0000 7f5ee57fa700 1 -- 172.21.15.40:0/2815700770 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]=0 set pool 4 allow_ec_overwrites to true v87) v1 ==== 169+0+0 (secure 0 0 0) 0x7f5ee0099be0 con 0x7f5ee8101660 2023-07-14T11:09:47.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.609+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 msgr2=0x7f5ed40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:47.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.609+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 0x7f5ed40a3680 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f5ee8100b30 tx=0x7f5ed8009260 comp rx=0 tx=0).stop 2023-07-14T11:09:47.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.609+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 msgr2=0x7f5ee8072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:47.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.609+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f5ee000efc0 tx=0x7f5ee000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 shutdown_connections 2023-07-14T11:09:47.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5ed40a11d0 0x7f5ed40a3680 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ee8101660 0x7f5ee8072e40 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ee80ffae0 0x7f5ee80742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 --2- 172.21.15.40:0/2815700770 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ee8100cd0 0x7f5ee8072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.614+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 >> 172.21.15.40:0/2815700770 conn(0x7f5ee80fb0e0 msgr2=0x7f5ee80fd3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:47.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.620+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 shutdown_connections 2023-07-14T11:09:47.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.621+0000 7f5eee1ff700 1 -- 172.21.15.40:0/2815700770 wait complete. 2023-07-14T11:09:47.629 INFO:teuthology.orchestra.run.smithi040.stderr:set pool 4 allow_ec_overwrites to true 2023-07-14T11:09:47.648 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs add_data_pool cephfs cephfs_data_ec 2023-07-14T11:09:47.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2885230983 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180ffae0 msgr2=0x7fec180ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2885230983 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180ffae0 0x7fec180ffef0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7fec08009990 tx=0x7fec08009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2885230983 shutdown_connections 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2885230983 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18101660 0x7fec18107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2885230983 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180ffae0 0x7fec180ffef0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2885230983 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec18100cd0 0x7fec18101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2885230983 >> 172.21.15.40:0/2885230983 conn(0x7fec180fb0e0 msgr2=0x7fec180fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:47.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2885230983 shutdown_connections 2023-07-14T11:09:47.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.854+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2885230983 wait complete. 2023-07-14T11:09:47.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 Processor -- start 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 -- start start 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 0x7fec18197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18100cd0 0x7fec18197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1819f5e0 con 0x7fec18101660 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1819f720 con 0x7fec180ffae0 2023-07-14T11:09:47.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.855+0000 7fec1e1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec1819f8b0 con 0x7fec18100cd0 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec177fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 0x7fec18197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35430/0 (socket says 172.21.15.40:35430) 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 -- 172.21.15.40:0/2077414319 learned_addr learned my addr 172.21.15.40:0/2077414319 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec16ffd700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18100cd0 0x7fec18197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:47.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 -- 172.21.15.40:0/2077414319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18100cd0 msgr2=0x7fec18197fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18100cd0 0x7fec18197fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 -- 172.21.15.40:0/2077414319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 msgr2=0x7fec18197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 0x7fec18197a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 -- 172.21.15.40:0/2077414319 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec080095f0 con 0x7fec18101660 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec177fe700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 0x7fec18197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:47.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec17fff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fec10007ee0 tx=0x7fec1000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:47.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.856+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec10017070 con 0x7fec18101660 2023-07-14T11:09:47.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.857+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec1819fb90 con 0x7fec18101660 2023-07-14T11:09:47.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.857+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fec1000f040 con 0x7fec18101660 2023-07-14T11:09:47.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.857+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fec10005550 con 0x7fec18101660 2023-07-14T11:09:47.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.857+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec181a00a0 con 0x7fec18101660 2023-07-14T11:09:47.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.858+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec18192170 con 0x7fec18101660 2023-07-14T11:09:47.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.861+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fec1000db20 con 0x7fec18101660 2023-07-14T11:09:47.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.863+0000 7fec14ff9700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 0x7fec000a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:47.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.863+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(87..87 src has 1..87) v4 ==== 8698+0+0 (secure 0 0 0) 0x7fec100cc870 con 0x7fec18101660 2023-07-14T11:09:47.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.863+0000 7fec177fe700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 0x7fec000a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:47.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.863+0000 7fec177fe700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 0x7fec000a3600 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fec0800b2e0 tx=0x7fec08005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:47.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:47.865+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fec10095d90 con 0x7fec18101660 2023-07-14T11:09:48.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:47 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-07-14T11:09:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:47 smithi130 ceph-mon[129405]: osdmap e86: 12 total, 12 up, 12 in 2023-07-14T11:09:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:47 smithi130 ceph-mon[129405]: pgmap v276: 129 pgs: 19 creating+peering, 67 active+clean, 43 unknown; 578 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 7 op/s 2023-07-14T11:09:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:47 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-07-14T11:09:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:47 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-07-14T11:09:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:47 smithi040 ceph-mon[97954]: osdmap e86: 12 total, 12 up, 12 in 2023-07-14T11:09:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:47 smithi040 ceph-mon[97954]: pgmap v276: 129 pgs: 19 creating+peering, 67 active+clean, 43 unknown; 578 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 7 op/s 2023-07-14T11:09:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:47 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-07-14T11:09:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:47 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data_ec", "pool_type": "erasure", "erasure_code_profile": "cephfs_data_ec", "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2023-07-14T11:09:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:47 smithi045 ceph-mon[128791]: osdmap e86: 12 total, 12 up, 12 in 2023-07-14T11:09:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:47 smithi045 ceph-mon[128791]: pgmap v276: 129 pgs: 19 creating+peering, 67 active+clean, 43 unknown; 578 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 1023 B/s wr, 7 op/s 2023-07-14T11:09:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:47 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]: dispatch 2023-07-14T11:09:48.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.124+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"} v 0) v1 -- 0x7fec1802c850 con 0x7fec18101660 2023-07-14T11:09:48.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.603+0000 7fec14ff9700 1 -- 172.21.15.40:0/2077414319 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]=0 added data pool 4 to fsmap v16) v1 ==== 137+0+0 (secure 0 0 0) 0x7fec100993b0 con 0x7fec18101660 2023-07-14T11:09:48.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 msgr2=0x7fec000a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 0x7fec000a3600 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fec0800b2e0 tx=0x7fec08005960 comp rx=0 tx=0).stop 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 msgr2=0x7fec1819f010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7fec10007ee0 tx=0x7fec1000d3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 shutdown_connections 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fec000a1150 0x7fec000a3600 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec18101660 0x7fec1819f010 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fec180ffae0 0x7fec18197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 --2- 172.21.15.40:0/2077414319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fec18100cd0 0x7fec18197fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 >> 172.21.15.40:0/2077414319 conn(0x7fec180fb0e0 msgr2=0x7fec18106620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 shutdown_connections 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.607+0000 7fec1e1ff700 1 -- 172.21.15.40:0/2077414319 wait complete. 2023-07-14T11:09:48.611 INFO:teuthology.orchestra.run.smithi040.stderr:added data pool 4 to fsmap 2023-07-14T11:09:48.633 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:48.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 -- 172.21.15.40:0/840465960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a0101680 msgr2=0x7fa7a0107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:48.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/840465960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a0101680 0x7fa7a0107f40 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7fa79800b0a0 tx=0x7fa79800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 -- 172.21.15.40:0/840465960 shutdown_connections 2023-07-14T11:09:48.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/840465960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a0101680 0x7fa7a0107f40 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/840465960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a00ffb00 0x7fa7a00fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/840465960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 -- 172.21.15.40:0/840465960 >> 172.21.15.40:0/840465960 conn(0x7fa7a00fb0e0 msgr2=0x7fa7a00fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:48.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 -- 172.21.15.40:0/840465960 shutdown_connections 2023-07-14T11:09:48.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.838+0000 7fa7a53ff700 1 -- 172.21.15.40:0/840465960 wait complete. 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.839+0000 7fa7a53ff700 1 Processor -- start 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.839+0000 7fa7a53ff700 1 -- start start 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a0101680 0x7fa7a019f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:48.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a019f6c0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a019f800 con 0x7fa7a0101680 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa7a53ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7a019f990 con 0x7fa7a0100cf0 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35450/0 (socket says 172.21.15.40:35450) 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 -- 172.21.15.40:0/2122889091 learned_addr learned my addr 172.21.15.40:0/2122889091 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79f7fe700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a0101680 0x7fa7a019f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa797fff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:48.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 -- 172.21.15.40:0/2122889091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 msgr2=0x7fa7a0198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 -- 172.21.15.40:0/2122889091 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a0101680 msgr2=0x7fa7a019f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a0101680 0x7fa7a019f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa79effd700 1 -- 172.21.15.40:0/2122889091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa798009d00 con 0x7fa7a00ffb00 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.840+0000 7fa797fff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa79effd700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa79000e970 tx=0x7fa79000ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa790009790 con 0x7fa7a00ffb00 2023-07-14T11:09:48.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7a019fc70 con 0x7fa7a00ffb00 2023-07-14T11:09:48.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa79000eec0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa790003cf0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.841+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7a01a01d0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.842+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa7a010b2e0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.844+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa79000ccd0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.846+0000 7fa79cff9700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 0x7fa78c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:48.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.846+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa790014070 con 0x7fa7a00ffb00 2023-07-14T11:09:48.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.846+0000 7fa797fff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 0x7fa78c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:48.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.846+0000 7fa797fff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 0x7fa78c0a3650 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fa788009c90 tx=0x7fa788009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:48.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:48.849+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa7900962b0 con 0x7fa7a00ffb00 2023-07-14T11:09:48.958 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:48 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-07-14T11:09:48.959 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:48 smithi045 ceph-mon[128791]: osdmap e87: 12 total, 12 up, 12 in 2023-07-14T11:09:48.959 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:48 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-07-14T11:09:49.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-07-14T11:09:49.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:48 smithi130 ceph-mon[129405]: osdmap e87: 12 total, 12 up, 12 in 2023-07-14T11:09:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:48 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-07-14T11:09:49.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:48 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2815700770' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "cephfs_data_ec", "var": "allow_ec_overwrites", "val": "true"}]': finished 2023-07-14T11:09:49.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:48 smithi040 ceph-mon[97954]: osdmap e87: 12 total, 12 up, 12 in 2023-07-14T11:09:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:48 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd=[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]: dispatch 2023-07-14T11:09:49.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.104+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fa7a019fe00 con 0x7fa7a00ffb00 2023-07-14T11:09:49.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.105+0000 7fa79cff9700 1 -- 172.21.15.40:0/2122889091 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 16 v16) v1 ==== 94+0+10779 (secure 0 0 0) 0x7fa7a019fe00 con 0x7fa7a00ffb00 2023-07-14T11:09:49.108 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:49.108 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":16,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24553,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24565,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":16,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:09:48.598409+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24569},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1},"id":1}]} 2023-07-14T11:09:49.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.107+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 msgr2=0x7fa78c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.107+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 0x7fa78c0a3650 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fa788009c90 tx=0x7fa788009260 comp rx=0 tx=0).stop 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.107+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 msgr2=0x7fa7a0197b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.107+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7fa79000e970 tx=0x7fa79000ec80 comp rx=0 tx=0).stop 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 shutdown_connections 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa78c0a11a0 0x7fa78c0a3650 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7a0101680 0x7fa7a019f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7a00ffb00 0x7fa7a0197b40 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 --2- 172.21.15.40:0/2122889091 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7a0100cf0 0x7fa7a0198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 >> 172.21.15.40:0/2122889091 conn(0x7fa7a00fb0e0 msgr2=0x7fa7a0106510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:49.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 shutdown_connections 2023-07-14T11:09:49.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.108+0000 7fa7a53ff700 1 -- 172.21.15.40:0/2122889091 wait complete. 2023-07-14T11:09:49.112 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 16 2023-07-14T11:09:49.129 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:49.313 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:09:48 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:09:48.954+0000 7f3cfd495700 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 -- 172.21.15.40:0/3511187519 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84580ffb00 msgr2=0x7f84580fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 --2- 172.21.15.40:0/3511187519 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84580ffb00 0x7f84580fff10 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f844c009990 tx=0x7f844c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 -- 172.21.15.40:0/3511187519 shutdown_connections 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 --2- 172.21.15.40:0/3511187519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458101680 0x7f8458107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 --2- 172.21.15.40:0/3511187519 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84580ffb00 0x7f84580fff10 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 --2- 172.21.15.40:0/3511187519 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8458100cf0 0x7f8458101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 -- 172.21.15.40:0/3511187519 >> 172.21.15.40:0/3511187519 conn(0x7f84580fb0e0 msgr2=0x7f84580fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:49.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 -- 172.21.15.40:0/3511187519 shutdown_connections 2023-07-14T11:09:49.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.330+0000 7f845ebff700 1 -- 172.21.15.40:0/3511187519 wait complete. 2023-07-14T11:09:49.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.331+0000 7f845ebff700 1 Processor -- start 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.331+0000 7f845ebff700 1 -- start start 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84580ffb00 0x7f8458197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f845819f5f0 con 0x7f8458100cf0 2023-07-14T11:09:49.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f845819f730 con 0x7f8458101680 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845ebff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f845819f8c0 con 0x7f84580ffb00 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f8457fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39654/0 (socket says 172.21.15.40:39654) 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845c99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84580ffb00 0x7f8458197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f8457fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35462/0 (socket says 172.21.15.40:35462) 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 -- 172.21.15.40:0/2297729574 learned_addr learned my addr 172.21.15.40:0/2297729574 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:49.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 -- 172.21.15.40:0/2297729574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84580ffb00 msgr2=0x7f8458197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84580ffb00 0x7f8458197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 -- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 msgr2=0x7f8458197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f845d19c700 1 -- 172.21.15.40:0/2297729574 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f844c0095f0 con 0x7f8458101680 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.332+0000 7f8457fff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f845d19c700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f845000ef30 tx=0x7f845000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8450018070 con 0x7f8458101680 2023-07-14T11:09:49.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f845000f040 con 0x7f8458101680 2023-07-14T11:09:49.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f845819fb40 con 0x7f8458101680 2023-07-14T11:09:49.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f845000cd00 con 0x7f8458101680 2023-07-14T11:09:49.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.333+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84581a0050 con 0x7f8458101680 2023-07-14T11:09:49.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.334+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f84581921e0 con 0x7f8458101680 2023-07-14T11:09:49.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.336+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8450003a00 con 0x7f8458101680 2023-07-14T11:09:49.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.338+0000 7f8455ffb700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 0x7f84480a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.338+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f84500cc800 con 0x7f8458101680 2023-07-14T11:09:49.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.338+0000 7f845c99b700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 0x7f84480a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.339+0000 7f845c99b700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 0x7f84480a3650 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f844c00b2e0 tx=0x7f844c005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:49.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.343+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8450095d90 con 0x7f8458101680 2023-07-14T11:09:49.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.558+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f845802c850 con 0x7f8458101680 2023-07-14T11:09:49.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.561+0000 7f8455ffb700 1 -- 172.21.15.40:0/2297729574 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v88) v1 ==== 74+0+23899 (secure 0 0 0) 0x7f845004dd70 con 0x7f8458101680 2023-07-14T11:09:49.563 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:49.564 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":88,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:09:48.126425+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 msgr2=0x7f84480a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 0x7f84480a3650 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f844c00b2e0 tx=0x7f844c005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 msgr2=0x7f845819f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f845000ef30 tx=0x7f845000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 shutdown_connections 2023-07-14T11:09:49.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84480a11a0 0x7f84480a3650 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8458101680 0x7f845819f020 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84580ffb00 0x7f8458197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 --2- 172.21.15.40:0/2297729574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8458100cf0 0x7f8458197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 >> 172.21.15.40:0/2297729574 conn(0x7f84580fb0e0 msgr2=0x7f84581065e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 shutdown_connections 2023-07-14T11:09:49.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.563+0000 7f845ebff700 1 -- 172.21.15.40:0/2297729574 wait complete. 2023-07-14T11:09:49.584 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:49.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.785+0000 7fea401ff700 1 -- 172.21.15.40:0/3260451290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 msgr2=0x7fea380fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.785+0000 7fea401ff700 1 --2- 172.21.15.40:0/3260451290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380fff10 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fea340098f0 tx=0x7fea34009c00 comp rx=0 tx=0).stop 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 -- 172.21.15.40:0/3260451290 shutdown_connections 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 --2- 172.21.15.40:0/3260451290 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 0x7fea38107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 --2- 172.21.15.40:0/3260451290 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380fff10 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 --2- 172.21.15.40:0/3260451290 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 -- 172.21.15.40:0/3260451290 >> 172.21.15.40:0/3260451290 conn(0x7fea380fb0e0 msgr2=0x7fea380fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 -- 172.21.15.40:0/3260451290 shutdown_connections 2023-07-14T11:09:49.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.786+0000 7fea401ff700 1 -- 172.21.15.40:0/3260451290 wait complete. 2023-07-14T11:09:49.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 Processor -- start 2023-07-14T11:09:49.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 -- start start 2023-07-14T11:09:49.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 0x7fea38072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea380735e0 con 0x7fea38100cf0 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea38073750 con 0x7fea38101680 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea401ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea3819f890 con 0x7fea380ffb00 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea3df9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea3d79a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea3df9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49416/0 (socket says 172.21.15.40:49416) 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea3df9b700 1 -- 172.21.15.40:0/4183362113 learned_addr learned my addr 172.21.15.40:0/4183362113 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.787+0000 7fea3e79c700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 0x7fea38072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 -- 172.21.15.40:0/4183362113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 msgr2=0x7fea380742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 -- 172.21.15.40:0/4183362113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 msgr2=0x7fea38072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 0x7fea38072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:49.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 -- 172.21.15.40:0/4183362113 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea280095f0 con 0x7fea38100cf0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3df9b700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea3d79a700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38072900 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fea2800eb90 tx=0x7fea2800eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea28009970 con 0x7fea38100cf0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea34009520 con 0x7fea38100cf0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea280044e0 con 0x7fea38100cf0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea28003e50 con 0x7fea38100cf0 2023-07-14T11:09:49.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.788+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea3819ff40 con 0x7fea38100cf0 2023-07-14T11:09:49.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.789+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea3810b2c0 con 0x7fea38100cf0 2023-07-14T11:09:49.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.791+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fea28004000 con 0x7fea38100cf0 2023-07-14T11:09:49.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.793+0000 7fea2effd700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 0x7fea240a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:49.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.793+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fea28014070 con 0x7fea38100cf0 2023-07-14T11:09:49.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.793+0000 7fea3df9b700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 0x7fea240a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:49.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.794+0000 7fea3df9b700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 0x7fea240a3650 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fea3400c010 tx=0x7fea34019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:49.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:49.797+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fea28096510 con 0x7fea38100cf0 2023-07-14T11:09:50.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.006+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fea381a05e0 con 0x7fea38100cf0 2023-07-14T11:09:50.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.008+0000 7fea2effd700 1 -- 172.21.15.40:0/4183362113 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v88) v1 ==== 74+0+23899 (secure 0 0 0) 0x7fea28099b30 con 0x7fea38100cf0 2023-07-14T11:09:50.011 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:50.011 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":88,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:09:48.126425+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:50.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 msgr2=0x7fea240a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 0x7fea240a3650 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fea3400c010 tx=0x7fea34019040 comp rx=0 tx=0).stop 2023-07-14T11:09:50.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 msgr2=0x7fea38072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38072900 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7fea2800eb90 tx=0x7fea2800eea0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 shutdown_connections 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea240a11a0 0x7fea240a3650 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea38101680 0x7fea38072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea380ffb00 0x7fea380742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 --2- 172.21.15.40:0/4183362113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea38100cf0 0x7fea38072900 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.010+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 >> 172.21.15.40:0/4183362113 conn(0x7fea380fb0e0 msgr2=0x7fea380fd320 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:50.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.011+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 shutdown_connections 2023-07-14T11:09:50.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.011+0000 7fea401ff700 1 -- 172.21.15.40:0/4183362113 wait complete. 2023-07-14T11:09:50.030 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: osdmap e88: 12 total, 12 up, 12 in 2023-07-14T11:09:50.030 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-07-14T11:09:50.030 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:50.031 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: pgmap v279: 129 pgs: 12 creating+peering, 99 active+clean, 18 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s wr, 8 op/s 2023-07-14T11:09:50.031 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2122889091' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.031 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2297729574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.051 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:09:50.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: osdmap e88: 12 total, 12 up, 12 in 2023-07-14T11:09:50.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-07-14T11:09:50.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:50.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: pgmap v279: 129 pgs: 12 creating+peering, 99 active+clean, 18 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s wr, 8 op/s 2023-07-14T11:09:50.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2122889091' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.053 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2297729574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: osdmap e88: 12 total, 12 up, 12 in 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2077414319' entity='client.admin' cmd='[{"prefix": "fs add_data_pool", "fs_name": "cephfs", "pool": "cephfs_data_ec"}]': finished 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: pgmap v279: 129 pgs: 12 creating+peering, 99 active+clean, 18 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.7 KiB/s wr, 8 op/s 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2122889091' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2297729574' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 -- 172.21.15.40:0/1529878513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93800ffb00 msgr2=0x7f93800fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 --2- 172.21.15.40:0/1529878513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93800ffb00 0x7f93800fff10 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7f937c009990 tx=0x7f937c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 -- 172.21.15.40:0/1529878513 shutdown_connections 2023-07-14T11:09:50.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 --2- 172.21.15.40:0/1529878513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 0x7f9380107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 --2- 172.21.15.40:0/1529878513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93800ffb00 0x7f93800fff10 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 --2- 172.21.15.40:0/1529878513 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9380100cf0 0x7f9380101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 -- 172.21.15.40:0/1529878513 >> 172.21.15.40:0/1529878513 conn(0x7f93800fb0e0 msgr2=0x7f93800fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:50.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 -- 172.21.15.40:0/1529878513 shutdown_connections 2023-07-14T11:09:50.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.253+0000 7f93877ff700 1 -- 172.21.15.40:0/1529878513 wait complete. 2023-07-14T11:09:50.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 Processor -- start 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 -- start start 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 0x7f938019cdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f938019d3c0 con 0x7f9380100cf0 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f938019d500 con 0x7f93800ffb00 2023-07-14T11:09:50.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.254+0000 7f93877ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f938019d690 con 0x7f9380101680 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f938559b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35482/0 (socket says 172.21.15.40:35482) 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f938559b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39694/0 (socket says 172.21.15.40:39694) 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 -- 172.21.15.40:0/335327049 learned_addr learned my addr 172.21.15.40:0/335327049 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9385d9c700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 0x7f938019cdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 -- 172.21.15.40:0/335327049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 msgr2=0x7f938019cdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 0x7f938019cdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 -- 172.21.15.40:0/335327049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 msgr2=0x7f9380195840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 -- 172.21.15.40:0/335327049 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f937c0095f0 con 0x7f9380100cf0 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f938559b700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f9384d9a700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f937000e9a0 tx=0x7f937000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.255+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9370009790 con 0x7f9380100cf0 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.256+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f938019d970 con 0x7f9380100cf0 2023-07-14T11:09:50.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.256+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9370010430 con 0x7f9380100cf0 2023-07-14T11:09:50.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.256+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9370018680 con 0x7f9380100cf0 2023-07-14T11:09:50.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.256+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f938019de80 con 0x7f9380100cf0 2023-07-14T11:09:50.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.257+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f938010b320 con 0x7f9380100cf0 2023-07-14T11:09:50.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.259+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f93700105a0 con 0x7f9380100cf0 2023-07-14T11:09:50.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.260+0000 7f93767fc700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 0x7f936c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.260+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f9370014070 con 0x7f9380100cf0 2023-07-14T11:09:50.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.261+0000 7f938559b700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 0x7f936c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.261+0000 7f938559b700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 0x7f936c0a3650 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f937c00b2e0 tx=0x7f937c005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:50.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.264+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9370051790 con 0x7f9380100cf0 2023-07-14T11:09:50.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.519+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f938010b4e0 con 0x7f9380100cf0 2023-07-14T11:09:50.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.520+0000 7f93767fc700 1 -- 172.21.15.40:0/335327049 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 16 v16) v1 ==== 94+0+10779 (secure 0 0 0) 0x7f9370096220 con 0x7f9380100cf0 2023-07-14T11:09:50.523 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:50.523 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":16,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7},{"gid":24521,"name":"h","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":9},{"gid":24523,"name":"c","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":8},{"gid":24541,"name":"i","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":10},{"gid":24553,"name":"g","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":11},{"gid":24565,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":12}],"filesystems":[{"mdsmap":{"epoch":16,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:09:48.598409+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24569},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":1},"id":1}]} 2023-07-14T11:09:50.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.522+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 msgr2=0x7f936c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.522+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 0x7f936c0a3650 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f937c00b2e0 tx=0x7f937c005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.522+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 msgr2=0x7f9380195d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.522+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7f937000e9a0 tx=0x7f937000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 shutdown_connections 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f936c0a11a0 0x7f936c0a3650 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9380101680 0x7f938019cdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93800ffb00 0x7f9380195840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 --2- 172.21.15.40:0/335327049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9380100cf0 0x7f9380195d80 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 >> 172.21.15.40:0/335327049 conn(0x7f93800fb0e0 msgr2=0x7f93800fd310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:50.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 shutdown_connections 2023-07-14T11:09:50.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.523+0000 7f93877ff700 1 -- 172.21.15.40:0/335327049 wait complete. 2023-07-14T11:09:50.527 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 16 2023-07-14T11:09:50.544 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:50.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3456309460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c40ffb00 msgr2=0x7fa9c40fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3456309460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40fff10 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fa9b4009990 tx=0x7fa9b4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3456309460 shutdown_connections 2023-07-14T11:09:50.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3456309460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4101680 0x7fa9c4107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3456309460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40fff10 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3456309460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c4100cf0 0x7fa9c4101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3456309460 >> 172.21.15.40:0/3456309460 conn(0x7fa9c40fb0e0 msgr2=0x7fa9c40fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:50.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3456309460 shutdown_connections 2023-07-14T11:09:50.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.749+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3456309460 wait complete. 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 Processor -- start 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 -- start start 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4100cf0 0x7fa9c4072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9c40735e0 con 0x7fa9c4101680 2023-07-14T11:09:50.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9c4073750 con 0x7fa9c40ffb00 2023-07-14T11:09:50.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.751+0000 7fa9c9dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9c419f8a0 con 0x7fa9c4100cf0 2023-07-14T11:09:50.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35494/0 (socket says 172.21.15.40:35494) 2023-07-14T11:09:50.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 -- 172.21.15.40:0/3854650419 learned_addr learned my addr 172.21.15.40:0/3854650419 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:50.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c37fe700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c2ffd700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4100cf0 0x7fa9c4072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 -- 172.21.15.40:0/3854650419 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4100cf0 msgr2=0x7fa9c4072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4100cf0 0x7fa9c4072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 -- 172.21.15.40:0/3854650419 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c40ffb00 msgr2=0x7fa9c40742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 -- 172.21.15.40:0/3854650419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa9b40095f0 con 0x7fa9c4101680 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.752+0000 7fa9c3fff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fa9bc007ee0 tx=0x7fa9bc00d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.753+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9bc017070 con 0x7fa9c4101680 2023-07-14T11:09:50.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.753+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9c419fb80 con 0x7fa9c4101680 2023-07-14T11:09:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.753+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa9bc00f040 con 0x7fa9c4101680 2023-07-14T11:09:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.753+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa9bc005580 con 0x7fa9c4101680 2023-07-14T11:09:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.753+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9c41a0090 con 0x7fa9c4101680 2023-07-14T11:09:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.754+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa9c410b320 con 0x7fa9c4101680 2023-07-14T11:09:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.756+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa9bc00db20 con 0x7fa9c4101680 2023-07-14T11:09:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.758+0000 7fa9c0ff9700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 0x7fa9ac0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.758+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa9bc0cc260 con 0x7fa9c4101680 2023-07-14T11:09:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.758+0000 7fa9c37fe700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 0x7fa9ac0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.759+0000 7fa9c37fe700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 0x7fa9ac0a3650 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fa9b400b5c0 tx=0x7fa9b4005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:50.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.762+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa9bc095760 con 0x7fa9c4101680 2023-07-14T11:09:50.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4183362113' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/335327049' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:50.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.974+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fa9c41069a0 con 0x7fa9c4101680 2023-07-14T11:09:50.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.976+0000 7fa9c0ff9700 1 -- 172.21.15.40:0/3854650419 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v88) v1 ==== 74+0+23899 (secure 0 0 0) 0x7fa9bc098d80 con 0x7fa9c4101680 2023-07-14T11:09:50.979 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:50.979 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":88,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:09:48.126425+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:50.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.978+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 msgr2=0x7fa9ac0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.978+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 0x7fa9ac0a3650 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fa9b400b5c0 tx=0x7fa9b4005fd0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 msgr2=0x7fa9c4072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:50.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fa9bc007ee0 tx=0x7fa9bc00d3b0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 shutdown_connections 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa9ac0a11a0 0x7fa9ac0a3650 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9c4101680 0x7fa9c4072e40 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa9c40ffb00 0x7fa9c40742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 --2- 172.21.15.40:0/3854650419 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa9c4100cf0 0x7fa9c4072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:50.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 >> 172.21.15.40:0/3854650419 conn(0x7fa9c40fb0e0 msgr2=0x7fa9c40fd320 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:50.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 shutdown_connections 2023-07-14T11:09:50.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:50.979+0000 7fa9c9dff700 1 -- 172.21.15.40:0/3854650419 wait complete. 2023-07-14T11:09:51.000 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 16, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2023-07-14T11:09:43.127642+0000', 'modified': '2023-07-14T11:09:48.598409+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24569}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24569': {'gid': 24569, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.45:6839/180621035', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6838', 'nonce': 180621035}, {'type': 'v1', 'addr': '172.21.15.45:6839', 'nonce': 180621035}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 1} 2023-07-14T11:09:51.001 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2023-07-14T11:09:51.001 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2023-07-14T11:09:51.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4183362113' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:51.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/335327049' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4183362113' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/335327049' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:09:51.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 -- 172.21.15.40:0/2307980263 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 msgr2=0x7fa54c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 --2- 172.21.15.40:0/2307980263 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c101120 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fa534009990 tx=0x7fa534009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 -- 172.21.15.40:0/2307980263 shutdown_connections 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 --2- 172.21.15.40:0/2307980263 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 0x7fa54c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 --2- 172.21.15.40:0/2307980263 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 0x7fa54c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 --2- 172.21.15.40:0/2307980263 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c101120 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 -- 172.21.15.40:0/2307980263 >> 172.21.15.40:0/2307980263 conn(0x7fa54c0fb0e0 msgr2=0x7fa54c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 -- 172.21.15.40:0/2307980263 shutdown_connections 2023-07-14T11:09:51.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.203+0000 7fa550fff700 1 -- 172.21.15.40:0/2307980263 wait complete. 2023-07-14T11:09:51.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 Processor -- start 2023-07-14T11:09:51.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 -- start start 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 0x7fa54c197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 0x7fa54c19f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c19f5f0 con 0x7fa54c100cd0 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c19f730 con 0x7fa54c101660 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.204+0000 7fa550fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa54c19f8c0 con 0x7fa54c0ffae0 2023-07-14T11:09:51.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa54a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 0x7fa54c197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35512/0 (socket says 172.21.15.40:35512) 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 -- 172.21.15.40:0/3272160205 learned_addr learned my addr 172.21.15.40:0/3272160205 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 0x7fa54c19f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 -- 172.21.15.40:0/3272160205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 msgr2=0x7fa54c197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 0x7fa54c197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 -- 172.21.15.40:0/3272160205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 msgr2=0x7fa54c19f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 0x7fa54c19f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 -- 172.21.15.40:0/3272160205 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa5340095f0 con 0x7fa54c100cd0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa543fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fa53400b620 tx=0x7fa534005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.205+0000 7fa5437fe700 1 -- 172.21.15.40:0/3272160205 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53401d070 con 0x7fa54c100cd0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.206+0000 7fa5437fe700 1 -- 172.21.15.40:0/3272160205 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa534022470 con 0x7fa54c100cd0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.206+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa54c19fb40 con 0x7fa54c100cd0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.206+0000 7fa5437fe700 1 -- 172.21.15.40:0/3272160205 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa534017650 con 0x7fa54c100cd0 2023-07-14T11:09:51.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.206+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa54c19ffc0 con 0x7fa54c100cd0 2023-07-14T11:09:51.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.209+0000 7fa5437fe700 1 -- 172.21.15.40:0/3272160205 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa5340225e0 con 0x7fa54c100cd0 2023-07-14T11:09:51.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.210+0000 7fa5437fe700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 0x7fa5380a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.210+0000 7fa56484bb80 1 Processor -- start 2023-07-14T11:09:51.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.210+0000 7fa5437fe700 1 -- 172.21.15.40:0/3272160205 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa5340cdef0 con 0x7fa54c100cd0 2023-07-14T11:09:51.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.210+0000 7fa56484bb80 1 -- start start 2023-07-14T11:09:51.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.210+0000 7fa54a59c700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 0x7fa5380a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cac890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cae090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83bb1c60 con 0x55df83cac480 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83c049f0 con 0x55df83cacdd0 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa56484bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83bfde80 con 0x55df83cadc20 2023-07-14T11:09:51.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39732/0 (socket says 172.21.15.40:39732) 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 -- 172.21.15.40:0/1102822251 learned_addr learned my addr 172.21.15.40:0/1102822251 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa54a59c700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cac890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cae090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 -- 172.21.15.40:0/1102822251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 msgr2=0x55df83cae090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cae090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 -- 172.21.15.40:0/1102822251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 msgr2=0x55df83cac890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cac890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa543fff700 1 -- 172.21.15.40:0/1102822251 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55df83a2bd90 con 0x55df83cacdd0 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa54a59c700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cac890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.211+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cae090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.212+0000 7fa54a59c700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 0x7fa5380a3680 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa54c100b30 tx=0x7fa53c011720 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.212+0000 7fa543fff700 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fa53409b130 tx=0x7fa534009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7cb4f38224ebf363 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.212+0000 7fa540ff9700 1 -- 172.21.15.40:0/1102822251 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa5340a1070 con 0x55df83cacdd0 2023-07-14T11:09:51.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.212+0000 7fa540ff9700 1 -- 172.21.15.40:0/1102822251 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa53409f1d0 con 0x55df83cacdd0 2023-07-14T11:09:51.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.212+0000 7fa540ff9700 1 -- 172.21.15.40:0/1102822251 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53409a8f0 con 0x55df83cacdd0 2023-07-14T11:09:51.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- 172.21.15.40:0/1102822251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 msgr2=0x55df83cad690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fa53409b130 tx=0x7fa534009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- 172.21.15.40:0/1102822251 shutdown_connections 2023-07-14T11:09:51.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cae090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cac890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1102822251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cad690 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- 172.21.15.40:0/1102822251 >> 172.21.15.40:0/1102822251 conn(0x55df83ca1d50 msgr2=0x55df83ca7180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- 172.21.15.40:0/1102822251 shutdown_connections 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- 172.21.15.40:0/1102822251 wait complete. 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.213+0000 7fa56484bb80 1 -- start start 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 Processor -- start 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 0x55df83cb5db0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 0x55df83cbd330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83b9da70 con 0x55df83cadc20 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83b9d770 con 0x55df83cacdd0 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83a818d0 con 0x55df83cac480 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa56484bb80 10 client.0 _ll_register_callbacks cb 0x55df83b9b710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa54ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 0x55df83cbd330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.214+0000 7fa543fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 0x55df83cbd330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35534/0 (socket says 172.21.15.40:35534) 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 0x55df83cb5db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54ad9d700 1 -- 172.21.15.40:0/3155835 learned_addr learned my addr 172.21.15.40:0/3155835 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa543fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39744/0 (socket says 172.21.15.40:39744) 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 -- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 msgr2=0x55df83cb62f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 -- 172.21.15.40:0/3155835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 msgr2=0x55df83cbd330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 0x55df83cbd330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 -- 172.21.15.40:0/3155835 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa54c19ffc0 con 0x55df83cac480 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa543fff700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa54a59c700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 0x55df83cb5db0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa53c01e970 tx=0x7fa53c0047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4098cd99dd6243ba server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.215+0000 7fa531ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53c004220 con 0x55df83cac480 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55df83b574a0 con 0x55df83cac480 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa53c004380 con 0x55df83cac480 2023-07-14T11:09:51.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=16) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa53c004540 con 0x55df83cac480 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 10 client.24682 fetch_fsmap learned FSMap version 16 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=16,monmap=4+}) v3 -- 0x55df83a14c10 con 0x55df83cac480 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55df83b574a0 con 0x55df83cac480 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 16) v1 ==== 7482+0+0 (secure 0 0 0) 0x7fa53c003af0 con 0x55df83cac480 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 10 client.24682 fetch_fsmap finished waiting for FSMap version 16 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 10 client.24682 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 20 client.24682 populate_metadata read hostname 'smithi040' 2023-07-14T11:09:51.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.216+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83cc1130 0x55df83cc04e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.217+0000 7fa56484bb80 4 client.24682 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:09:51.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.217+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55df83bf6e60 con 0x55df83cc1130 2023-07-14T11:09:51.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.217+0000 7fa543fff700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83cc1130 0x55df83cc04e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.217+0000 7fa543fff700 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83cc1130 0x55df83cc04e0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a129c4caf5c93489 server_cookie=b728e6d05ff232d6 in_seq=0 out_seq=0 2023-07-14T11:09:51.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.217+0000 7fa531ffb700 10 client.24682 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:09:51.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.218+0000 7fa531ffb700 1 -- 172.21.15.40:0/3155835 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55df83bf6e60 con 0x55df83cc1130 2023-07-14T11:09:51.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.218+0000 7fa531ffb700 10 client.24682 handle_command_reply: tid=0 2023-07-14T11:09:51.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.218+0000 7fa56484bb80 1 client.24682 shutdown 2023-07-14T11:09:51.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.218+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 msgr2=0x55df83cb5db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.218+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 0x55df83cb5db0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fa53c01e970 tx=0x7fa53c0047c0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 shutdown_connections 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83cc1130 0x55df83cc04e0 crc :-1 s=READY pgs=13 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cadc20 0x55df83cbd330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cac480 0x55df83cb5db0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 --2- 172.21.15.40:0/3155835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cb62f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa531ffb700 0 client.24682 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:09:51.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 >> 172.21.15.40:0/3155835 conn(0x55df83ca1d50 msgr2=0x55df83ca6860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 shutdown_connections 2023-07-14T11:09:51.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 1 -- 172.21.15.40:0/3155835 wait complete. 2023-07-14T11:09:51.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.219+0000 7fa56484bb80 20 client.24682 trim_cache size 0 max 16384 2023-07-14T11:09:51.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 Processor -- start 2023-07-14T11:09:51.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 -- start start 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cc0a20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbd2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83b574a0 con 0x55df83cac480 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83bcc790 con 0x55df83cacdd0 2023-07-14T11:09:51.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.237+0000 7fa56484bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83a8dde0 con 0x55df83cadc20 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35536/0 (socket says 172.21.15.40:35536) 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 -- 172.21.15.40:0/452600910 learned_addr learned my addr 172.21.15.40:0/452600910 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa543fff700 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cc0a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbd2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 -- 172.21.15.40:0/452600910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 msgr2=0x55df83cbd2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbd2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 -- 172.21.15.40:0/452600910 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 msgr2=0x55df83cc0a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cc0a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 -- 172.21.15.40:0/452600910 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55df83a52200 con 0x55df83cac480 2023-07-14T11:09:51.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.238+0000 7fa54a59c700 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fa53c0097a0 tx=0x7fa53c009990 comp rx=0 tx=0).ready entity=mon.0 client_cookie=7840ef057d16abaf server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa5417fa700 1 -- 172.21.15.40:0/452600910 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53c0041d0 con 0x55df83cac480 2023-07-14T11:09:51.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa5417fa700 1 -- 172.21.15.40:0/452600910 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa53c00e4d0 con 0x55df83cac480 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa5417fa700 1 -- 172.21.15.40:0/452600910 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa53c00fac0 con 0x55df83cac480 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 -- 172.21.15.40:0/452600910 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 msgr2=0x55df83cc04e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7fa53c0097a0 tx=0x7fa53c009990 comp rx=0 tx=0).stop 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 -- 172.21.15.40:0/452600910 shutdown_connections 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbd2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cac480 0x55df83cc04e0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 --2- 172.21.15.40:0/452600910 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cacdd0 0x55df83cc0a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 -- 172.21.15.40:0/452600910 >> 172.21.15.40:0/452600910 conn(0x55df83ca1d50 msgr2=0x55df83cb4070 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 -- 172.21.15.40:0/452600910 shutdown_connections 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.239+0000 7fa56484bb80 1 -- 172.21.15.40:0/452600910 wait complete. 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 -- start start 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 Processor -- start 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cac480 0x55df83cba3e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83cbbe70 con 0x55df83cacdd0 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83cbbfe0 con 0x55df83cac480 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55df83cbc1a0 con 0x55df83cadc20 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.240+0000 7fa56484bb80 10 client.0 _ll_register_callbacks cb 0x7fa53c01e760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa543fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa543fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35550/0 (socket says 172.21.15.40:35550) 2023-07-14T11:09:51.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49512/0 (socket says 172.21.15.40:49512) 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa543fff700 1 -- 172.21.15.40:0/1451760587 learned_addr learned my addr 172.21.15.40:0/1451760587 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54a59c700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cac480 0x55df83cba3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 -- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cac480 msgr2=0x55df83cba3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cac480 0x55df83cba3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 -- 172.21.15.40:0/1451760587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 msgr2=0x55df83cba920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 -- 172.21.15.40:0/1451760587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55df83a14c10 con 0x55df83cadc20 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa543fff700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa54ad9d700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x55df83cc0f80 tx=0x7fa544007f40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8c603011cc693517 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.241+0000 7fa531ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa54400e040 con 0x55df83cadc20 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa544004db0 con 0x55df83cadc20 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa53409f1d0 con 0x55df83cadc20 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=16) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa544018a70 con 0x55df83cadc20 2023-07-14T11:09:51.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 10 client.24688 fetch_fsmap learned FSMap version 16 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=16,monmap=4+}) v3 -- 0x55df83a2a260 con 0x55df83cadc20 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa54402b020 con 0x55df83cadc20 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 16) v1 ==== 7482+0+0 (secure 0 0 0) 0x7fa5440203f0 con 0x55df83cadc20 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 10 client.24688 fetch_fsmap finished waiting for FSMap version 16 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 10 client.24688 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.242+0000 7fa56484bb80 20 client.24688 populate_metadata read hostname 'smithi040' 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83d1a780 0x55df83d170f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:51.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa56484bb80 4 client.24688 mds_command: new command op to 24569 tid=0[{"prefix": "status"}] 2023-07-14T11:09:51.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55df83bb9000 con 0x55df83d1a780 2023-07-14T11:09:51.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa54a59c700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83d1a780 0x55df83d170f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa531ffb700 10 client.24688 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.243+0000 7fa54a59c700 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83d1a780 0x55df83d170f0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=92cc331aed2ff8ed server_cookie=390f710ee552192a in_seq=0 out_seq=0 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa531ffb700 1 -- 172.21.15.40:0/1451760587 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+326 (crc 0 0 0) 0x55df83bb9000 con 0x55df83d1a780 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa531ffb700 10 client.24688 handle_command_reply: tid=0 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 client.24688 shutdown 2023-07-14T11:09:51.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 msgr2=0x55df83cbb930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x55df83cc0f80 tx=0x7fa544007f40 comp rx=0 tx=0).stop 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 shutdown_connections 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55df83d1a780 0x55df83d170f0 crc :-1 s=READY pgs=14 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55df83cadc20 0x55df83cbb930 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55df83cac480 0x55df83cba3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa531ffb700 0 client.24688 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 --2- 172.21.15.40:0/1451760587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55df83cacdd0 0x55df83cba920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 >> 172.21.15.40:0/1451760587 conn(0x55df83ca1d50 msgr2=0x55df83cc04e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 shutdown_connections 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 1 -- 172.21.15.40:0/1451760587 wait complete. 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.244+0000 7fa56484bb80 20 client.24688 trim_cache size 0 max 16384 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:09:51.255 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 0, 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24569, 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 7.6287628449999998, 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 16, 2023-07-14T11:09:51.256 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 88, 2023-07-14T11:09:51.257 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 87, 2023-07-14T11:09:51.257 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 13.165468726 2023-07-14T11:09:51.257 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:09:51.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 msgr2=0x7fa5380a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 0x7fa5380a3680 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7fa54c100b30 tx=0x7fa53c011720 comp rx=0 tx=0).stop 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 msgr2=0x7fa54c197fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fa53400b620 tx=0x7fa534005740 comp rx=0 tx=0).stop 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 shutdown_connections 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa5380a11d0 0x7fa5380a3680 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa54c101660 0x7fa54c19f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa54c0ffae0 0x7fa54c197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 --2- 172.21.15.40:0/3272160205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa54c100cd0 0x7fa54c197fb0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:51.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 >> 172.21.15.40:0/3272160205 conn(0x7fa54c0fb0e0 msgr2=0x7fa54c0fd3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:51.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 shutdown_connections 2023-07-14T11:09:51.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:51.245+0000 7fa550fff700 1 -- 172.21.15.40:0/3272160205 wait complete. 2023-07-14T11:09:51.265 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2023-07-14T11:09:51.266 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:51.267 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph fs ls 2023-07-14T11:09:51.992 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:09:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:51 smithi130 ceph-mon[129405]: pgmap v280: 129 pgs: 12 creating+peering, 105 active+clean, 12 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2023-07-14T11:09:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:51 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3854650419' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:51 smithi040 ceph-mon[97954]: pgmap v280: 129 pgs: 12 creating+peering, 105 active+clean, 12 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2023-07-14T11:09:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:51 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3854650419' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:51 smithi045 ceph-mon[128791]: pgmap v280: 129 pgs: 12 creating+peering, 105 active+clean, 12 unknown; 579 KiB data, 328 MiB used, 1.0 TiB / 1.0 TiB avail; 2.2 KiB/s wr, 6 op/s 2023-07-14T11:09:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:51 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3854650419' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:09:53.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 -- 172.21.15.40:0/1458565415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f41007a0 msgr2=0x7fd2f4104770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:53.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 --2- 172.21.15.40:0/1458565415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f41007a0 0x7fd2f4104770 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7fd2e80099c0 tx=0x7fd2e8009cd0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 -- 172.21.15.40:0/1458565415 shutdown_connections 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 --2- 172.21.15.40:0/1458565415 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f4104ea0 0x7fd2f410ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 --2- 172.21.15.40:0/1458565415 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f40ffe00 0x7fd2f41001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 --2- 172.21.15.40:0/1458565415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f41007a0 0x7fd2f4104770 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 -- 172.21.15.40:0/1458565415 >> 172.21.15.40:0/1458565415 conn(0x7fd2f40fb660 msgr2=0x7fd2f40fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 -- 172.21.15.40:0/1458565415 shutdown_connections 2023-07-14T11:09:53.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.618+0000 7fd2fa561700 1 -- 172.21.15.40:0/1458565415 wait complete. 2023-07-14T11:09:53.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.620+0000 7fd2fa561700 1 Processor -- start 2023-07-14T11:09:53.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.620+0000 7fd2fa561700 1 -- start start 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f41007a0 0x7fd2f4072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 0x7fd2f4073290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd2f41a40c0 con 0x7fd2f4104ea0 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd2f41a4230 con 0x7fd2f40ffe00 2023-07-14T11:09:53.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2fa561700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd2f41a43f0 con 0x7fd2f41007a0 2023-07-14T11:09:53.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:53.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f8afe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 0x7fd2f4073290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:43922/0 (socket says 172.21.15.40:43922) 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 -- 172.21.15.40:0/3935041963 learned_addr learned my addr 172.21.15.40:0/3935041963 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f37fe700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f41007a0 0x7fd2f4072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 -- 172.21.15.40:0/3935041963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f41007a0 msgr2=0x7fd2f4072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f41007a0 0x7fd2f4072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 -- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 msgr2=0x7fd2f4073290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:53.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 0x7fd2f4073290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f3fff700 1 -- 172.21.15.40:0/3935041963 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd2e80095f0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.621+0000 7fd2f8afe700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 0x7fd2f4073290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2f3fff700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fd2e400e9a0 tx=0x7fd2e400ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2e4009790 con 0x7fd2f40ffe00 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd2f41a46d0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd2e4010430 con 0x7fd2f40ffe00 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd2e4018680 con 0x7fd2f40ffe00 2023-07-14T11:09:53.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.622+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd2f41a4bf0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.623+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd2f404e210 con 0x7fd2f40ffe00 2023-07-14T11:09:53.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.625+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fd2e40105a0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.627+0000 7fd2f17fa700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 0x7fd2dc0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:53.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.627+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fd2e4014070 con 0x7fd2f40ffe00 2023-07-14T11:09:53.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.627+0000 7fd2f37fe700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 0x7fd2dc0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:53.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.627+0000 7fd2f37fe700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 0x7fd2dc0a3600 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fd2e8009990 tx=0x7fd2e8005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:53.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.629+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fd2e40961d0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.881+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7fd2f41a4e10 con 0x7fd2f40ffe00 2023-07-14T11:09:53.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.881+0000 7fd2f17fa700 1 -- 172.21.15.40:0/3935041963 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v16) v1 ==== 53+0+88 (secure 0 0 0) 0x7fd2e40997f0 con 0x7fd2f40ffe00 2023-07-14T11:09:53.885 INFO:teuthology.orchestra.run.smithi040.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ec ] 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 msgr2=0x7fd2dc0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 0x7fd2dc0a3600 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7fd2e8009990 tx=0x7fd2e8005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 msgr2=0x7fd2f4072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fd2e400e9a0 tx=0x7fd2e400ecb0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 shutdown_connections 2023-07-14T11:09:53.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd2dc0a1150 0x7fd2dc0a3600 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd2f4104ea0 0x7fd2f4073290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd2f40ffe00 0x7fd2f4072810 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 --2- 172.21.15.40:0/3935041963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd2f41007a0 0x7fd2f4072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 >> 172.21.15.40:0/3935041963 conn(0x7fd2f40fb660 msgr2=0x7fd2f40fc0d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 shutdown_connections 2023-07-14T11:09:53.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:53.884+0000 7fd2fa561700 1 -- 172.21.15.40:0/3935041963 wait complete. 2023-07-14T11:09:53.990 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:53 smithi040 ceph-mon[97954]: pgmap v281: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2023-07-14T11:09:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:53 smithi130 ceph-mon[129405]: pgmap v281: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2023-07-14T11:09:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:53 smithi045 ceph-mon[128791]: pgmap v281: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s wr, 2 op/s 2023-07-14T11:09:54.272 INFO:tasks.cephfs.mount:Mounting Ceph FS cephfs; just confirmed its presence on cluster 2023-07-14T11:09:54.273 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2023-07-14T11:09:54.273 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi040.front.sepia.ceph.com 2023-07-14T11:09:54.273 INFO:tasks.cephfs.mount:self.client.name = client.admin 2023-07-14T11:09:54.273 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:54.274 INFO:tasks.cephfs.mount:self.cephfs_name = cephfs 2023-07-14T11:09:54.274 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2023-07-14T11:09:54.274 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2023-07-14T11:09:54.274 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.admin' 2023-07-14T11:09:54.275 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:54.275 DEBUG:teuthology.orchestra.run.smithi040:> ip addr 2023-07-14T11:09:54.321 INFO:teuthology.orchestra.run.smithi040.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: inet 127.0.0.1/8 scope host lo 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: inet6 ::1/128 scope host 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:09:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:6b:fd:16 brd ff:ff:ff:ff:ff:ff 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp7s0f0 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:6b:fd:17 brd ff:ff:ff:ff:ff:ff 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp7s0f1 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:bd:12:cc brd ff:ff:ff:ff:ff:ff 2023-07-14T11:09:54.323 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp1s0f0 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: inet 172.21.15.40/20 brd 172.21.15.255 scope global dynamic noprefixroute ens1f0 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft 41507sec preferred_lft 41507sec 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: inet6 fe80::ec4:7aff:febd:12cc/64 scope link 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout:5: ens1f1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:bd:12:cd brd ff:ff:ff:ff:ff:ff 2023-07-14T11:09:54.324 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp1s0f1 2023-07-14T11:09:54.326 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2023-07-14T11:09:54.326 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:54.326 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:09:54.327 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:09:54.327 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link add name ceph-brx type bridge 2023-07-14T11:09:54.327 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip addr flush dev ceph-brx 2023-07-14T11:09:54.328 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set ceph-brx up 2023-07-14T11:09:54.328 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2023-07-14T11:09:54.328 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:09:54.785 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:54.785 DEBUG:teuthology.orchestra.run.smithi040:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2023-07-14T11:09:54.859 INFO:teuthology.orchestra.run.smithi040.stdout:1 2023-07-14T11:09:54.862 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:54.862 DEBUG:teuthology.orchestra.run.smithi040:> route 2023-07-14T11:09:54.926 INFO:teuthology.orchestra.run.smithi040.stdout:Kernel IP routing table 2023-07-14T11:09:54.926 INFO:teuthology.orchestra.run.smithi040.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2023-07-14T11:09:54.926 INFO:teuthology.orchestra.run.smithi040.stdout:default _gateway 0.0.0.0 UG 100 0 0 ens1f0 2023-07-14T11:09:54.927 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 ens1f0 2023-07-14T11:09:54.927 INFO:teuthology.orchestra.run.smithi040.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2023-07-14T11:09:54.928 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:54.928 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:09:54.928 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:09:54.929 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -A FORWARD -o ens1f0 -i ceph-brx -j ACCEPT 2023-07-14T11:09:54.929 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -A FORWARD -i ens1f0 -o ceph-brx -j ACCEPT 2023-07-14T11:09:54.929 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o ens1f0 -j MASQUERADE 2023-07-14T11:09:54.930 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:09:55.033 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:54 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3935041963' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:09:55.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:54 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3935041963' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:09:55.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:54 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3935041963' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:09:55.639 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:55.639 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list 2023-07-14T11:09:55.684 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:55.684 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list-id 2023-07-14T11:09:55.768 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:55.769 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:09:55.769 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:09:55.769 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.admin 2023-07-14T11:09:55.769 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.admin 0 2023-07-14T11:09:55.770 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:09:55.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:55 smithi040 ceph-mon[97954]: pgmap v282: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 1 op/s 2023-07-14T11:09:55.956 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.admin' with 192.168.0.1/16 2023-07-14T11:09:55.957 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:55.957 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:09:55.957 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:09:55.958 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.admin type veth peer name brx.0 2023-07-14T11:09:55.958 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2023-07-14T11:09:55.958 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set veth0 up 2023-07-14T11:09:55.959 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip link set lo up 2023-07-14T11:09:55.959 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.admin ip route add default via 192.168.255.254 2023-07-14T11:09:55.959 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:09:56.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:55 smithi130 ceph-mon[129405]: pgmap v282: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 1 op/s 2023-07-14T11:09:56.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:55 smithi045 ceph-mon[128791]: pgmap v282: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.2 KiB/s wr, 1 op/s 2023-07-14T11:09:56.241 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:56.242 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:09:56.242 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:09:56.242 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set brx.0 up 2023-07-14T11:09:56.243 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set dev brx.0 master ceph-brx 2023-07-14T11:09:56.243 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:09:56.420 INFO:tasks.cephfs.fuse_mount:Client client.admin config is {} 2023-07-14T11:09:56.420 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-07-14T11:09:56.420 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -v /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:56.465 INFO:teuthology.orchestra.run.smithi040.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.admin' 2023-07-14T11:09:56.465 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-07-14T11:09:56.466 DEBUG:teuthology.orchestra.run.smithi040:> chmod 0000 /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:56.549 DEBUG:teuthology.orchestra.run.smithi040:> sudo modprobe fuse 2023-07-14T11:09:56.660 INFO:teuthology.orchestra.run:Running command with timeout 30 2023-07-14T11:09:56.661 DEBUG:teuthology.orchestra.run.smithi040:> sudo mount -t fusectl /sys/fs/fuse/connections /sys/fs/fuse/connections 2023-07-14T11:09:56.730 INFO:teuthology.orchestra.run.smithi040.stderr:mount: /sys/fs/fuse/connections: /sys/fs/fuse/connections already mounted or mount point busy. 2023-07-14T11:09:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 32 2023-07-14T11:09:56.734 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:56.734 DEBUG:teuthology.orchestra.run.smithi040:> ls /sys/fs/fuse/connections 2023-07-14T11:09:56.777 INFO:tasks.cephfs.fuse_mount:Pre-mount connections: [] 2023-07-14T11:09:56.778 DEBUG:teuthology.orchestra.run.smithi040:> (cd /home/ubuntu/cephtest && exec sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.admin sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' /home/ubuntu/cephtest/mnt.admin --id admin --client_fs=cephfs) 2023-07-14T11:09:56.821 DEBUG:teuthology.orchestra.run.smithi040:> sudo modprobe fuse 2023-07-14T11:09:56.893 INFO:teuthology.orchestra.run:Running command with timeout 30 2023-07-14T11:09:56.893 DEBUG:teuthology.orchestra.run.smithi040:> sudo mount -t fusectl /sys/fs/fuse/connections /sys/fs/fuse/connections 2023-07-14T11:09:56.964 INFO:teuthology.orchestra.run.smithi040.stderr:mount: /sys/fs/fuse/connections: /sys/fs/fuse/connections already mounted or mount point busy. 2023-07-14T11:09:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 32 2023-07-14T11:09:56.968 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:56.968 DEBUG:teuthology.orchestra.run.smithi040:> ls /sys/fs/fuse/connections 2023-07-14T11:09:57.029 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi040.stderr:2023-07-14T11:09:57.027+0000 7fbe689f75c0 -1 init, newargv = 0x557d34bd3e20 newargc=15 2023-07-14T11:09:57.030 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi040.stderr:ceph-fuse[134663]: starting ceph client 2023-07-14T11:09:57.039 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi040.stderr:ceph-fuse[134663]: starting fuse 2023-07-14T11:09:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:57 smithi130 ceph-mon[129405]: pgmap v283: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:09:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:57 smithi040 ceph-mon[97954]: pgmap v283: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:09:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:57 smithi045 ceph-mon[128791]: pgmap v283: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:09:58.114 DEBUG:teuthology.orchestra.run.smithi040:> sudo modprobe fuse 2023-07-14T11:09:58.194 INFO:teuthology.orchestra.run:Running command with timeout 30 2023-07-14T11:09:58.194 DEBUG:teuthology.orchestra.run.smithi040:> sudo mount -t fusectl /sys/fs/fuse/connections /sys/fs/fuse/connections 2023-07-14T11:09:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:mount: /sys/fs/fuse/connections: /sys/fs/fuse/connections already mounted or mount point busy. 2023-07-14T11:09:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 32 2023-07-14T11:09:58.268 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:58.269 DEBUG:teuthology.orchestra.run.smithi040:> ls /sys/fs/fuse/connections 2023-07-14T11:09:58.312 INFO:teuthology.orchestra.run.smithi040.stdout:121 2023-07-14T11:09:58.313 INFO:tasks.cephfs.fuse_mount:Post-mount connections: [121] 2023-07-14T11:09:58.313 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits daemon-helper kill python3 -c ' 2023-07-14T11:09:58.313 DEBUG:teuthology.orchestra.run.smithi040:> import glob 2023-07-14T11:09:58.314 DEBUG:teuthology.orchestra.run.smithi040:> import re 2023-07-14T11:09:58.314 DEBUG:teuthology.orchestra.run.smithi040:> import os 2023-07-14T11:09:58.314 DEBUG:teuthology.orchestra.run.smithi040:> import subprocess 2023-07-14T11:09:58.314 DEBUG:teuthology.orchestra.run.smithi040:> 2023-07-14T11:09:58.315 DEBUG:teuthology.orchestra.run.smithi040:> def _find_admin_socket(client_name): 2023-07-14T11:09:58.315 DEBUG:teuthology.orchestra.run.smithi040:> asok_path = "/var/run/ceph/ceph-client.admin.*.asok" 2023-07-14T11:09:58.315 DEBUG:teuthology.orchestra.run.smithi040:> files = glob.glob(asok_path) 2023-07-14T11:09:58.315 DEBUG:teuthology.orchestra.run.smithi040:> mountpoint = "/home/ubuntu/cephtest/mnt.admin" 2023-07-14T11:09:58.316 DEBUG:teuthology.orchestra.run.smithi040:> 2023-07-14T11:09:58.316 DEBUG:teuthology.orchestra.run.smithi040:> # Given a non-glob path, it better be there 2023-07-14T11:09:58.316 DEBUG:teuthology.orchestra.run.smithi040:> if "*" not in asok_path: 2023-07-14T11:09:58.317 DEBUG:teuthology.orchestra.run.smithi040:> assert(len(files) == 1) 2023-07-14T11:09:58.317 DEBUG:teuthology.orchestra.run.smithi040:> return files[0] 2023-07-14T11:09:58.317 DEBUG:teuthology.orchestra.run.smithi040:> 2023-07-14T11:09:58.317 DEBUG:teuthology.orchestra.run.smithi040:> for f in files: 2023-07-14T11:09:58.318 DEBUG:teuthology.orchestra.run.smithi040:> pid = re.match(".*\.(\d+)\.asok$", f).group(1) 2023-07-14T11:09:58.318 DEBUG:teuthology.orchestra.run.smithi040:> if os.path.exists("/proc/{0}".format(pid)): 2023-07-14T11:09:58.318 DEBUG:teuthology.orchestra.run.smithi040:> with open("/proc/{0}/cmdline".format(pid), '"'"'r'"'"') as proc_f: 2023-07-14T11:09:58.318 DEBUG:teuthology.orchestra.run.smithi040:> contents = proc_f.read() 2023-07-14T11:09:58.319 DEBUG:teuthology.orchestra.run.smithi040:> if mountpoint in contents: 2023-07-14T11:09:58.319 DEBUG:teuthology.orchestra.run.smithi040:> return f 2023-07-14T11:09:58.319 DEBUG:teuthology.orchestra.run.smithi040:> raise RuntimeError("Client socket {0} not found".format(client_name)) 2023-07-14T11:09:58.319 DEBUG:teuthology.orchestra.run.smithi040:> 2023-07-14T11:09:58.320 DEBUG:teuthology.orchestra.run.smithi040:> print(_find_admin_socket("client.admin")) 2023-07-14T11:09:58.320 DEBUG:teuthology.orchestra.run.smithi040:> ' 2023-07-14T11:09:58.499 INFO:teuthology.orchestra.run.smithi040.stdout:/var/run/ceph/ceph-client.admin.134663.asok 2023-07-14T11:09:58.676 INFO:tasks.cephfs.fuse_mount:Found client admin socket at /var/run/ceph/ceph-client.admin.134663.asok 2023-07-14T11:09:58.676 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:58.677 DEBUG:teuthology.orchestra.run.smithi040:> sudo ceph --admin-daemon /var/run/ceph/ceph-client.admin.134663.asok status 2023-07-14T11:09:58.908 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "metadata": { 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_sha1": "3fc0089cc1082313a98853d48b892acdc1385b8b", 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_version": "ceph version 18.0.0-4757-g3fc0089c (3fc0089cc1082313a98853d48b892acdc1385b8b) reef (dev)", 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "entity_id": "admin", 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "hostname": "smithi040", 2023-07-14T11:09:58.909 INFO:teuthology.orchestra.run.smithi040.stdout: "mount_point": "/home/ubuntu/cephtest/mnt.admin", 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "pid": "134663", 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "root": "/" 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "dentry_count": 0, 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "dentry_pinned_count": 0, 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 14694, 2023-07-14T11:09:58.910 INFO:teuthology.orchestra.run.smithi040.stdout: "inst": { 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "name": { 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "client", 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "num": 14694 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": { 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v1", 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "192.168.0.1:0", 2023-07-14T11:09:58.911 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 1075849483 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": { 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v1", 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "192.168.0.1:0", 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 1075849483 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:09:58.912 INFO:teuthology.orchestra.run.smithi040.stdout: "inst_str": "client.14694 192.168.0.1:0/1075849483", 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "addr_str": "192.168.0.1:0/1075849483", 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "inode_count": 1, 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_epoch": 16, 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_epoch": 88, 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_epoch_barrier": 0, 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "blocklisted": false, 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs" 2023-07-14T11:09:58.913 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:09:58.923 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:58.923 DEBUG:teuthology.orchestra.run.smithi040:> stat --file-system '--printf=%T 2023-07-14T11:09:58.923 DEBUG:teuthology.orchestra.run.smithi040:> ' -- /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:58.968 INFO:teuthology.orchestra.run.smithi040.stdout:fuseblk 2023-07-14T11:09:58.968 INFO:tasks.cephfs.fuse_mount:ceph-fuse is mounted on /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:58.969 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:58.969 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:09:59.080 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:09:59.080 DEBUG:teuthology.orchestra.run.smithi040:> (cd /home/ubuntu/cephtest/mnt.admin && exec bash -c 'setfattr -n ceph.dir.layout.pool -v cephfs_data_ec . && getfattr -n ceph.dir.layout .') 2023-07-14T11:09:59.211 INFO:teuthology.orchestra.run.smithi040.stdout:# file: . 2023-07-14T11:09:59.211 INFO:teuthology.orchestra.run.smithi040.stdout:ceph.dir.layout="stripe_unit=4194304 stripe_count=1 object_size=4194304 pool=cephfs_data_ec" 2023-07-14T11:09:59.211 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:59.212 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:09:59.213 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:09:59.256 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:09:59.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 -- 172.21.15.40:0/4250361265 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d8074550 msgr2=0x7fe0d80729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/4250361265 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d8074550 0x7fe0d80729a0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe0cc009990 tx=0x7fe0cc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 -- 172.21.15.40:0/4250361265 shutdown_connections 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/4250361265 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d81079c0 0x7fe0d8109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/4250361265 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d8074550 0x7fe0d80729a0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/4250361265 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8072f70 0x7fe0d80733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 -- 172.21.15.40:0/4250361265 >> 172.21.15.40:0/4250361265 conn(0x7fe0d80fb0e0 msgr2=0x7fe0d80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:59.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 -- 172.21.15.40:0/4250361265 shutdown_connections 2023-07-14T11:09:59.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.498+0000 7fe0df1ff700 1 -- 172.21.15.40:0/4250361265 wait complete. 2023-07-14T11:09:59.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 Processor -- start 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 -- start start 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d8072f70 0x7fe0d8197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8074550 0x7fe0d8197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0d819f620 con 0x7fe0d8072f70 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0d819f760 con 0x7fe0d81079c0 2023-07-14T11:09:59.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.499+0000 7fe0df1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0d819f8f0 con 0x7fe0d8074550 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dcf9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d8072f70 0x7fe0d8197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0d7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8074550 0x7fe0d8197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:43936/0 (socket says 172.21.15.40:43936) 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 -- 172.21.15.40:0/3761391006 learned_addr learned my addr 172.21.15.40:0/3761391006 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 -- 172.21.15.40:0/3761391006 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8074550 msgr2=0x7fe0d8197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8074550 0x7fe0d8197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 -- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d8072f70 msgr2=0x7fe0d8197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d8072f70 0x7fe0d8197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 -- 172.21.15.40:0/3761391006 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe0d0009c40 con 0x7fe0d81079c0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.500+0000 7fe0dd79c700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000c370 tx=0x7fe0d000c680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.501+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d0017070 con 0x7fe0d81079c0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.501+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe0cc0095f0 con 0x7fe0d81079c0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.501+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe0d000f040 con 0x7fe0d81079c0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.501+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe0d000e050 con 0x7fe0d81079c0 2023-07-14T11:09:59.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.501+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe0d819ff40 con 0x7fe0d81079c0 2023-07-14T11:09:59.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.502+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe0d8106070 con 0x7fe0d81079c0 2023-07-14T11:09:59.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.504+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fe0d0003a00 con 0x7fe0d81079c0 2023-07-14T11:09:59.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.505+0000 7fe0d5ffb700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 0x7fe0c80a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.505+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fe0d00cc250 con 0x7fe0d81079c0 2023-07-14T11:09:59.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.505+0000 7fe0dcf9b700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 0x7fe0c80a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.506+0000 7fe0dcf9b700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 0x7fe0c80a3650 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fe0cc00b2e0 tx=0x7fe0cc005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:59.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.507+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fe0d00957e0 con 0x7fe0d81079c0 2023-07-14T11:09:59.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.724+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fe0d802ca60 con 0x7fe0d81079c0 2023-07-14T11:09:59.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.727+0000 7fe0d5ffb700 1 -- 172.21.15.40:0/3761391006 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v88) v1 ==== 74+0+23899 (secure 0 0 0) 0x7fe0d0098e00 con 0x7fe0d81079c0 2023-07-14T11:09:59.730 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:09:59.730 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":88,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:09:48.126425+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:09:59.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 msgr2=0x7fe0c80a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 0x7fe0c80a3650 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fe0cc00b2e0 tx=0x7fe0cc005fb0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 msgr2=0x7fe0d819f050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7fe0d000c370 tx=0x7fe0d000c680 comp rx=0 tx=0).stop 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 shutdown_connections 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe0c80a11a0 0x7fe0c80a3650 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe0d81079c0 0x7fe0d819f050 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0d8072f70 0x7fe0d8197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 --2- 172.21.15.40:0/3761391006 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe0d8074550 0x7fe0d8197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 >> 172.21.15.40:0/3761391006 conn(0x7fe0d80fb0e0 msgr2=0x7fe0d80fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:59.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.729+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 shutdown_connections 2023-07-14T11:09:59.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.730+0000 7fe0df1ff700 1 -- 172.21.15.40:0/3761391006 wait complete. 2023-07-14T11:09:59.751 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:09:59.751 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:09:59.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 -- 172.21.15.40:0/3244181755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 msgr2=0x7efdfc0ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 --2- 172.21.15.40:0/3244181755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc0ffef0 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7efdf0009990 tx=0x7efdf0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 -- 172.21.15.40:0/3244181755 shutdown_connections 2023-07-14T11:09:59.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 --2- 172.21.15.40:0/3244181755 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 0x7efdfc107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 --2- 172.21.15.40:0/3244181755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc0ffef0 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 --2- 172.21.15.40:0/3244181755 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 0x7efdfc101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 -- 172.21.15.40:0/3244181755 >> 172.21.15.40:0/3244181755 conn(0x7efdfc0fb0e0 msgr2=0x7efdfc0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:09:59.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 -- 172.21.15.40:0/3244181755 shutdown_connections 2023-07-14T11:09:59.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.959+0000 7efe02bff700 1 -- 172.21.15.40:0/3244181755 wait complete. 2023-07-14T11:09:59.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 Processor -- start 2023-07-14T11:09:59.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 -- start start 2023-07-14T11:09:59.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 0x7efdfc197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 0x7efdfc19f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdfc19f610 con 0x7efdfc0ffae0 2023-07-14T11:09:59.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdfc19f750 con 0x7efdfc100cd0 2023-07-14T11:09:59.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.961+0000 7efe02bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdfc19f8e0 con 0x7efdfc101660 2023-07-14T11:09:59.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:53998/0 (socket says 172.21.15.40:53998) 2023-07-14T11:09:59.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 -- 172.21.15.40:0/4105867435 learned_addr learned my addr 172.21.15.40:0/4105867435 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:09:59.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efdfbfff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 0x7efdfc197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0119c700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 0x7efdfc19f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 -- 172.21.15.40:0/4105867435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 msgr2=0x7efdfc19f040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 0x7efdfc19f040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 -- 172.21.15.40:0/4105867435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 msgr2=0x7efdfc197fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 0x7efdfc197fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 -- 172.21.15.40:0/4105867435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efdf00095f0 con 0x7efdfc0ffae0 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.962+0000 7efe0099b700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7efdf00098e0 tx=0x7efdf0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.963+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdf001d070 con 0x7efdfc0ffae0 2023-07-14T11:09:59.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.963+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efdf00054f0 con 0x7efdfc0ffae0 2023-07-14T11:09:59.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.963+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efdfc19fb60 con 0x7efdfc0ffae0 2023-07-14T11:09:59.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.963+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efdf0004ca0 con 0x7efdfc0ffae0 2023-07-14T11:09:59.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.963+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efdfc19ff20 con 0x7efdfc0ffae0 2023-07-14T11:09:59.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.964+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efdfc1921c0 con 0x7efdfc0ffae0 2023-07-14T11:09:59.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.966+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7efdf000fb90 con 0x7efdfc0ffae0 2023-07-14T11:09:59.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.967+0000 7efdf9ffb700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 0x7efdec0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:09:59.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.967+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7efdf00cde50 con 0x7efdfc0ffae0 2023-07-14T11:09:59.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.968+0000 7efdfbfff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 0x7efdec0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:09:59.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.968+0000 7efdfbfff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 0x7efdec0a3600 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7efde8009c60 tx=0x7efde8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:09:59.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:09:59.971+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7efdf005a790 con 0x7efdfc0ffae0 2023-07-14T11:10:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:09:59 smithi130 ceph-mon[129405]: pgmap v284: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:10:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:09:59 smithi040 ceph-mon[97954]: pgmap v284: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:10:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:09:59 smithi045 ceph-mon[128791]: pgmap v284: 129 pgs: 129 active+clean; 579 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s wr, 0 op/s 2023-07-14T11:10:00.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.179+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7efdfc0611c0 con 0x7efdfc0ffae0 2023-07-14T11:10:00.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.179+0000 7efdf9ffb700 1 -- 172.21.15.40:0/4105867435 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v88) v1 ==== 93+0+10 (secure 0 0 0) 0x7efdf00973e0 con 0x7efdfc0ffae0 2023-07-14T11:10:00.182 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 msgr2=0x7efdec0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 0x7efdec0a3600 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7efde8009c60 tx=0x7efde8009260 comp rx=0 tx=0).stop 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 msgr2=0x7efdfc197a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7efdf00098e0 tx=0x7efdf0004730 comp rx=0 tx=0).stop 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 shutdown_connections 2023-07-14T11:10:00.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efdec0a1150 0x7efdec0a3600 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdfc101660 0x7efdfc19f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdfc0ffae0 0x7efdfc197a90 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 --2- 172.21.15.40:0/4105867435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdfc100cd0 0x7efdfc197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 >> 172.21.15.40:0/4105867435 conn(0x7efdfc0fb0e0 msgr2=0x7efdfc106600 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 shutdown_connections 2023-07-14T11:10:00.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.182+0000 7efe02bff700 1 -- 172.21.15.40:0/4105867435 wait complete. 2023-07-14T11:10:00.203 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/3777136448 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c58101660 msgr2=0x7f0c58107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/3777136448 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c58101660 0x7f0c58107f20 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f0c5000b0a0 tx=0x7f0c5000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/3777136448 shutdown_connections 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/3777136448 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c58101660 0x7f0c58107f20 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/3777136448 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c580ffae0 0x7f0c580ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/3777136448 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58100cd0 0x7f0c58101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/3777136448 >> 172.21.15.40:0/3777136448 conn(0x7f0c580fb0e0 msgr2=0x7f0c580fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:00.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/3777136448 shutdown_connections 2023-07-14T11:10:00.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.418+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/3777136448 wait complete. 2023-07-14T11:10:00.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 Processor -- start 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 -- start start 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c58100cd0 0x7f0c58198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 0x7f0c5819f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5819f680 con 0x7f0c580ffae0 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5819f7c0 con 0x7f0c58101660 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.419+0000 7f0c5d7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c5819f950 con 0x7f0c58100cd0 2023-07-14T11:10:00.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54010/0 (socket says 172.21.15.40:54010) 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 -- 172.21.15.40:0/1585412026 learned_addr learned my addr 172.21.15.40:0/1585412026 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c4ffff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c58100cd0 0x7f0c58198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c577fe700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 0x7f0c5819f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 -- 172.21.15.40:0/1585412026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c58100cd0 msgr2=0x7f0c58198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c58100cd0 0x7f0c58198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 -- 172.21.15.40:0/1585412026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 msgr2=0x7f0c5819f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 0x7f0c5819f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 -- 172.21.15.40:0/1585412026 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c40009520 con 0x7f0c580ffae0 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c577fe700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 0x7f0c5819f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.420+0000 7f0c56ffd700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f0c4800e970 tx=0x7f0c4800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.421+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c48009790 con 0x7f0c580ffae0 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.421+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0c4800eec0 con 0x7f0c580ffae0 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.421+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c50009d00 con 0x7f0c580ffae0 2023-07-14T11:10:00.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.421+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0c48003cf0 con 0x7f0c580ffae0 2023-07-14T11:10:00.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.421+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c581a0000 con 0x7f0c580ffae0 2023-07-14T11:10:00.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.422+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c5810b2c0 con 0x7f0c580ffae0 2023-07-14T11:10:00.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.424+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f0c4800ccd0 con 0x7f0c580ffae0 2023-07-14T11:10:00.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.425+0000 7f0c54ff9700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 0x7f0c440a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.425+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f0c48014070 con 0x7f0c580ffae0 2023-07-14T11:10:00.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.425+0000 7f0c4ffff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 0x7f0c440a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.426+0000 7f0c4ffff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 0x7f0c440a3680 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0c40009df0 tx=0x7f0c40009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:00.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.429+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0c48096210 con 0x7f0c580ffae0 2023-07-14T11:10:00.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.639+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f0c581a05b0 con 0x7f0c580ffae0 2023-07-14T11:10:00.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.639+0000 7f0c54ff9700 1 -- 172.21.15.40:0/1585412026 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v88) v1 ==== 104+0+11 (secure 0 0 0) 0x7f0c48099830 con 0x7f0c580ffae0 2023-07-14T11:10:00.642 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:10:00.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 msgr2=0x7f0c440a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 0x7f0c440a3680 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0c40009df0 tx=0x7f0c40009260 comp rx=0 tx=0).stop 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 msgr2=0x7f0c58197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f0c4800e970 tx=0x7f0c4800ec80 comp rx=0 tx=0).stop 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 shutdown_connections 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0c440a11d0 0x7f0c440a3680 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c58101660 0x7f0c5819f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c580ffae0 0x7f0c58197b00 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 --2- 172.21.15.40:0/1585412026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c58100cd0 0x7f0c58198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 >> 172.21.15.40:0/1585412026 conn(0x7f0c580fb0e0 msgr2=0x7f0c580fd280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:00.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 shutdown_connections 2023-07-14T11:10:00.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.642+0000 7f0c5d7ff700 1 -- 172.21.15.40:0/1585412026 wait complete. 2023-07-14T11:10:00.663 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:10:00.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 -- 172.21.15.40:0/2251388158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c100cd0 msgr2=0x7fb04c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 --2- 172.21.15.40:0/2251388158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c100cd0 0x7fb04c101120 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb034009990 tx=0x7fb034009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 -- 172.21.15.40:0/2251388158 shutdown_connections 2023-07-14T11:10:00.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 --2- 172.21.15.40:0/2251388158 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 0x7fb04c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 --2- 172.21.15.40:0/2251388158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c0ffae0 0x7fb04c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 --2- 172.21.15.40:0/2251388158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c100cd0 0x7fb04c101120 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 -- 172.21.15.40:0/2251388158 >> 172.21.15.40:0/2251388158 conn(0x7fb04c0fb0e0 msgr2=0x7fb04c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:00.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 -- 172.21.15.40:0/2251388158 shutdown_connections 2023-07-14T11:10:00.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.868+0000 7fb0519ff700 1 -- 172.21.15.40:0/2251388158 wait complete. 2023-07-14T11:10:00.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.869+0000 7fb0519ff700 1 Processor -- start 2023-07-14T11:10:00.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.869+0000 7fb0519ff700 1 -- start start 2023-07-14T11:10:00.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c100cd0 0x7fb04c195dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 0x7fb04c19ce30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb04c19d400 con 0x7fb04c0ffae0 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb04c19d540 con 0x7fb04c101660 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb0519ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb04c19d6d0 con 0x7fb04c100cd0 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54028/0 (socket says 172.21.15.40:54028) 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 -- 172.21.15.40:0/4199660165 learned_addr learned my addr 172.21.15.40:0/4199660165 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04b7fe700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 0x7fb04c19ce30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb043fff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c100cd0 0x7fb04c195dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 -- 172.21.15.40:0/4199660165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c100cd0 msgr2=0x7fb04c195dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c100cd0 0x7fb04c195dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 -- 172.21.15.40:0/4199660165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 msgr2=0x7fb04c19ce30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 0x7fb04c19ce30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:00.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.870+0000 7fb04affd700 1 -- 172.21.15.40:0/4199660165 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0340095f0 con 0x7fb04c0ffae0 2023-07-14T11:10:00.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb04affd700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fb03c00e970 tx=0x7fb03c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:00.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb03c009790 con 0x7fb04c0ffae0 2023-07-14T11:10:00.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb03c00c7e0 con 0x7fb04c0ffae0 2023-07-14T11:10:00.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb03c003c70 con 0x7fb04c0ffae0 2023-07-14T11:10:00.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb04c19d9b0 con 0x7fb04c0ffae0 2023-07-14T11:10:00.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.871+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb04c19de70 con 0x7fb04c0ffae0 2023-07-14T11:10:00.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.872+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb04c10b300 con 0x7fb04c0ffae0 2023-07-14T11:10:00.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.874+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb03c00cc50 con 0x7fb04c0ffae0 2023-07-14T11:10:00.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.877+0000 7fb0509fd700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 0x7fb0380a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:00.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.878+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fb03c014070 con 0x7fb04c0ffae0 2023-07-14T11:10:00.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.878+0000 7fb043fff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 0x7fb0380a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:00.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.878+0000 7fb043fff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 0x7fb0380a3680 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fb03400b2e0 tx=0x7fb034005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:00.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:00.881+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb03c096170 con 0x7fb04c0ffae0 2023-07-14T11:10:01.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:00 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3761391006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:00 smithi130 ceph-mon[129405]: overall HEALTH_OK 2023-07-14T11:10:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:00 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4105867435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:00 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1585412026' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:00 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3761391006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:00 smithi040 ceph-mon[97954]: overall HEALTH_OK 2023-07-14T11:10:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:00 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4105867435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:00 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1585412026' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:00 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3761391006' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:00 smithi045 ceph-mon[128791]: overall HEALTH_OK 2023-07-14T11:10:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:00 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4105867435' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:00 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1585412026' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:01.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.089+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fb04c19e4e0 con 0x7fb04c0ffae0 2023-07-14T11:10:01.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.089+0000 7fb0509fd700 1 -- 172.21.15.40:0/4199660165 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v88) v1 ==== 100+0+11 (secure 0 0 0) 0x7fb03c003dd0 con 0x7fb04c0ffae0 2023-07-14T11:10:01.092 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:10:01.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.091+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 msgr2=0x7fb0380a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.091+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 0x7fb0380a3680 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7fb03400b2e0 tx=0x7fb034005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.091+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 msgr2=0x7fb04c195880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.091+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7fb03c00e970 tx=0x7fb03c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 shutdown_connections 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb0380a11d0 0x7fb0380a3680 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb04c101660 0x7fb04c19ce30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb04c0ffae0 0x7fb04c195880 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 --2- 172.21.15.40:0/4199660165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb04c100cd0 0x7fb04c195dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 >> 172.21.15.40:0/4199660165 conn(0x7fb04c0fb0e0 msgr2=0x7fb04c0fd350 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:01.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 shutdown_connections 2023-07-14T11:10:01.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.092+0000 7fb0519ff700 1 -- 172.21.15.40:0/4199660165 wait complete. 2023-07-14T11:10:01.112 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:10:01.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 -- 172.21.15.40:0/1589297905 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 msgr2=0x7f4720107f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 --2- 172.21.15.40:0/1589297905 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f4720107f30 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f471800b0a0 tx=0x7f471800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 -- 172.21.15.40:0/1589297905 shutdown_connections 2023-07-14T11:10:01.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 --2- 172.21.15.40:0/1589297905 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f4720107f30 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 --2- 172.21.15.40:0/1589297905 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f47200ffaf0 0x7f47200fff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 --2- 172.21.15.40:0/1589297905 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4720100ce0 0x7f4720101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 -- 172.21.15.40:0/1589297905 >> 172.21.15.40:0/1589297905 conn(0x7f47200fb130 msgr2=0x7f47200fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:01.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 -- 172.21.15.40:0/1589297905 shutdown_connections 2023-07-14T11:10:01.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.317+0000 7f47265ff700 1 -- 172.21.15.40:0/1589297905 wait complete. 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.318+0000 7f47265ff700 1 Processor -- start 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.318+0000 7f47265ff700 1 -- start start 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.318+0000 7f47265ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f47265ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4720100ce0 0x7f47201980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f47265ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f472019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f47265ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f472019f6e0 con 0x7f4720101670 2023-07-14T11:10:01.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f47265ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f472019f820 con 0x7f47200ffaf0 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f47265ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f472019f9b0 con 0x7f4720100ce0 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f471ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f472019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f471ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:44040/0 (socket says 172.21.15.40:44040) 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f471ffff700 1 -- 172.21.15.40:0/1118335759 learned_addr learned my addr 172.21.15.40:0/1118335759 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4717fff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4720100ce0 0x7f47201980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 -- 172.21.15.40:0/1118335759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4720100ce0 msgr2=0x7f47201980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4720100ce0 0x7f47201980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 -- 172.21.15.40:0/1118335759 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 msgr2=0x7f4720197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f4724b9c700 1 -- 172.21.15.40:0/1118335759 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4718009d00 con 0x7f4720101670 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.319+0000 7f471ffff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f4724b9c700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f472019f110 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f471800b070 tx=0x7f471800bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f471800e040 con 0x7f4720101670 2023-07-14T11:10:01.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4718009480 con 0x7f4720101670 2023-07-14T11:10:01.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f47180052a0 con 0x7f4720101670 2023-07-14T11:10:01.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f472019fc30 con 0x7f4720101670 2023-07-14T11:10:01.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.320+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47201a0000 con 0x7f4720101670 2023-07-14T11:10:01.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.321+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f472010b320 con 0x7f4720101670 2023-07-14T11:10:01.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.323+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f471801d480 con 0x7f4720101670 2023-07-14T11:10:01.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.324+0000 7f471dffb700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 0x7f470c0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.325+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f47180cdfc0 con 0x7f4720101670 2023-07-14T11:10:01.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.325+0000 7f471ffff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 0x7f470c0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.325+0000 7f471ffff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 0x7f470c0a3680 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f4710005950 tx=0x7f4710009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:01.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.326+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4718097550 con 0x7f4720101670 2023-07-14T11:10:01.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.538+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f47201a0670 con 0x7f4720101670 2023-07-14T11:10:01.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.538+0000 7f471dffb700 1 -- 172.21.15.40:0/1118335759 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v88) v1 ==== 103+0+11 (secure 0 0 0) 0x7f4718004030 con 0x7f4720101670 2023-07-14T11:10:01.541 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:10:01.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.540+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 msgr2=0x7f470c0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 0x7f470c0a3680 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f4710005950 tx=0x7f4710009260 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 msgr2=0x7f472019f110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f472019f110 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f471800b070 tx=0x7f471800bc80 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 shutdown_connections 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f470c0a11d0 0x7f470c0a3680 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4720101670 0x7f472019f110 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47200ffaf0 0x7f4720197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 --2- 172.21.15.40:0/1118335759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4720100ce0 0x7f47201980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 >> 172.21.15.40:0/1118335759 conn(0x7f47200fb130 msgr2=0x7f47200fd290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:01.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 shutdown_connections 2023-07-14T11:10:01.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.541+0000 7f47265ff700 1 -- 172.21.15.40:0/1118335759 wait complete. 2023-07-14T11:10:01.562 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2023-07-14T11:10:01.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 -- 172.21.15.40:0/2933424529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f35880ffbd0 msgr2=0x7f3588100020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 --2- 172.21.15.40:0/2933424529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f35880ffbd0 0x7f3588100020 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f3574009990 tx=0x7f3574009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 -- 172.21.15.40:0/2933424529 shutdown_connections 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 --2- 172.21.15.40:0/2933424529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3588100560 0x7f3588106e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 --2- 172.21.15.40:0/2933424529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f35880fedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 --2- 172.21.15.40:0/2933424529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f35880ffbd0 0x7f3588100020 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 -- 172.21.15.40:0/2933424529 >> 172.21.15.40:0/2933424529 conn(0x7f35880fa000 msgr2=0x7f35880fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 -- 172.21.15.40:0/2933424529 shutdown_connections 2023-07-14T11:10:01.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.765+0000 7f358ebff700 1 -- 172.21.15.40:0/2933424529 wait complete. 2023-07-14T11:10:01.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.767+0000 7f358ebff700 1 Processor -- start 2023-07-14T11:10:01.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.767+0000 7f358ebff700 1 -- start start 2023-07-14T11:10:01.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.767+0000 7f358ebff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f3588102970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358ebff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358ebff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3588100560 0x7f35881014f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358ebff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3588101c90 con 0x7f35880ffbd0 2023-07-14T11:10:01.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358ebff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3588101e00 con 0x7f3588100560 2023-07-14T11:10:01.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358ebff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f358819f900 con 0x7f35880fe9e0 2023-07-14T11:10:01.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54076/0 (socket says 172.21.15.40:54076) 2023-07-14T11:10:01.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 -- 172.21.15.40:0/76834299 learned_addr learned my addr 172.21.15.40:0/76834299 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:01.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358d19c700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3588100560 0x7f35881014f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358c99b700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f3588102970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 -- 172.21.15.40:0/76834299 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 msgr2=0x7f3588102970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f3588102970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 -- 172.21.15.40:0/76834299 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3588100560 msgr2=0x7f35881014f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3588100560 0x7f35881014f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 -- 172.21.15.40:0/76834299 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f35740095f0 con 0x7f35880ffbd0 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f358c99b700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f3588102970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:01.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.768+0000 7f3587fff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f357400b620 tx=0x7f3574005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:01.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.769+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f357401d070 con 0x7f35880ffbd0 2023-07-14T11:10:01.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.769+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f358819fb80 con 0x7f35880ffbd0 2023-07-14T11:10:01.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.769+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3574022470 con 0x7f35880ffbd0 2023-07-14T11:10:01.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.769+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3574017650 con 0x7f35880ffbd0 2023-07-14T11:10:01.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.769+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35881a0000 con 0x7f35880ffbd0 2023-07-14T11:10:01.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.770+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f358810a200 con 0x7f35880ffbd0 2023-07-14T11:10:01.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.772+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f35740225e0 con 0x7f35880ffbd0 2023-07-14T11:10:01.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.773+0000 7f3585ffb700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 0x7f35780a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:01.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.773+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f35740ce060 con 0x7f35880ffbd0 2023-07-14T11:10:01.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.774+0000 7f358c99b700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 0x7f35780a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:01.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.774+0000 7f358c99b700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 0x7f35780a3600 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f35880ffa30 tx=0x7f357c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:01.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:01.777+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f35740975f0 con 0x7f35880ffbd0 2023-07-14T11:10:02.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.001+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f35881a0670 con 0x7f35880ffbd0 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.002+0000 7f3585ffb700 1 -- 172.21.15.40:0/76834299 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 10 entries v88) v1 ==== 81+0+589 (secure 0 0 0) 0x7f357409ac10 con 0x7f35880ffbd0 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3160212490 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/4083556243 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/2326174215 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:02.005 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/1897779716 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:02.006 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3378800182 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:02.006 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:02.006 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:02.006 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:02.006 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/500170929 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:02.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 msgr2=0x7f35780a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 0x7f35780a3600 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f35880ffa30 tx=0x7f357c009260 comp rx=0 tx=0).stop 2023-07-14T11:10:02.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 msgr2=0x7f3588100fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f357400b620 tx=0x7f3574005740 comp rx=0 tx=0).stop 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 shutdown_connections 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f35780a1150 0x7f35780a3600 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3588100560 0x7f35881014f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f35880fe9e0 0x7f3588102970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 --2- 172.21.15.40:0/76834299 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35880ffbd0 0x7f3588100fb0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.004+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 >> 172.21.15.40:0/76834299 conn(0x7f35880fa000 msgr2=0x7f35880676b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:02.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.005+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 shutdown_connections 2023-07-14T11:10:02.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.005+0000 7f358ebff700 1 -- 172.21.15.40:0/76834299 wait complete. 2023-07-14T11:10:02.009 INFO:teuthology.orchestra.run.smithi040.stderr:listed 10 entries 2023-07-14T11:10:02.023 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:01 smithi040 ceph-mon[97954]: pgmap v285: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:02.023 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:01 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4199660165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.023 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:01 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1118335759' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.025 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:10:02.025 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:10:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:01 smithi130 ceph-mon[129405]: pgmap v285: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:01 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4199660165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:01 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1118335759' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:01 smithi045 ceph-mon[128791]: pgmap v285: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:01 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4199660165' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:01 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1118335759' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:02.069 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:10:02.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 -- 172.21.15.40:0/4279293210 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8101570 msgr2=0x7f8ba8107e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 --2- 172.21.15.40:0/4279293210 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8101570 0x7f8ba8107e30 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f8ba000b0a0 tx=0x7f8ba000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 -- 172.21.15.40:0/4279293210 shutdown_connections 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 --2- 172.21.15.40:0/4279293210 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8101570 0x7f8ba8107e30 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 --2- 172.21.15.40:0/4279293210 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba80ffe00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 --2- 172.21.15.40:0/4279293210 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8100be0 0x7f8ba8101030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 -- 172.21.15.40:0/4279293210 >> 172.21.15.40:0/4279293210 conn(0x7f8ba80fb010 msgr2=0x7f8ba80fd440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 -- 172.21.15.40:0/4279293210 shutdown_connections 2023-07-14T11:10:02.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.310+0000 7f8bae39d700 1 -- 172.21.15.40:0/4279293210 wait complete. 2023-07-14T11:10:02.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.311+0000 7f8bae39d700 1 Processor -- start 2023-07-14T11:10:02.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 -- start start 2023-07-14T11:10:02.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba8197870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 0x7f8ba8197db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 0x7f8ba819ee20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ba819f3f0 con 0x7f8ba8101570 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ba819f530 con 0x7f8ba8100be0 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bae39d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ba819f6c0 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8badb9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 0x7f8ba819ee20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bacb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 0x7f8ba8197db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bad39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba8197870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bacb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 0x7f8ba8197db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:44074/0 (socket says 172.21.15.40:44074) 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.312+0000 7f8bacb9a700 1 -- 172.21.15.40:0/766293525 learned_addr learned my addr 172.21.15.40:0/766293525 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 -- 172.21.15.40:0/766293525 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 msgr2=0x7f8ba8197db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 0x7f8ba8197db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 -- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 msgr2=0x7f8ba819ee20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 0x7f8ba819ee20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 -- 172.21.15.40:0/766293525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b98009520 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8badb9c700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 0x7f8ba819ee20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bad39b700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba8197870 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f8ba400e9c0 tx=0x7f8ba400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ba4004220 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8ba0009d00 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8ba4004380 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8ba4003be0 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.313+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ba819fd70 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.314+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8ba810b220 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.317+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8ba4003d40 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.318+0000 7f8b9e7fc700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 0x7f8b940a36f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.318+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f8ba4014070 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.318+0000 7f8bacb9a700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 0x7f8b940a36f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.319+0000 7f8bacb9a700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 0x7f8b940a36f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8b98009ef0 tx=0x7f8b98009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:02.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.320+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8ba4096490 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.530+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f8ba810b3b0 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.532+0000 7f8b9e7fc700 1 -- 172.21.15.40:0/766293525 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v88) v1 ==== 74+0+23899 (secure 0 0 0) 0x7f8ba401f340 con 0x7f8ba80ff9f0 2023-07-14T11:10:02.535 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:02.536 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":88,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:09:48.126425+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:02.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 msgr2=0x7f8b940a36f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 0x7f8b940a36f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8b98009ef0 tx=0x7f8b98009260 comp rx=0 tx=0).stop 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 msgr2=0x7f8ba8197870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba8197870 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f8ba400e9c0 tx=0x7f8ba400ecd0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 shutdown_connections 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b940a1240 0x7f8b940a36f0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ba8101570 0x7f8ba819ee20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8ba80ff9f0 0x7f8ba8197870 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 --2- 172.21.15.40:0/766293525 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8ba8100be0 0x7f8ba8197db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 >> 172.21.15.40:0/766293525 conn(0x7f8ba80fb010 msgr2=0x7f8ba8072740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:02.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 shutdown_connections 2023-07-14T11:10:02.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.535+0000 7f8bae39d700 1 -- 172.21.15.40:0/766293525 wait complete. 2023-07-14T11:10:02.556 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:10:02.557 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:10:02.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/2348718460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c080ffae0 msgr2=0x7f0c080ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/2348718460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c080ffae0 0x7f0c080ffef0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f0bf8009990 tx=0x7f0bf8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/2348718460 shutdown_connections 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/2348718460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c08101660 0x7f0c08107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/2348718460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c080ffae0 0x7f0c080ffef0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/2348718460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 0x7f0c08101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/2348718460 >> 172.21.15.40:0/2348718460 conn(0x7f0c080fb0e0 msgr2=0x7f0c080fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/2348718460 shutdown_connections 2023-07-14T11:10:02.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.763+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/2348718460 wait complete. 2023-07-14T11:10:02.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.764+0000 7f0c0d1ff700 1 Processor -- start 2023-07-14T11:10:02.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.764+0000 7f0c0d1ff700 1 -- start start 2023-07-14T11:10:02.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 0x7f0c08195dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c08101660 0x7f0c0819ce40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c0819d410 con 0x7f0c080ffae0 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c0819d550 con 0x7f0c08101660 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0d1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c0819d6e0 con 0x7f0c08100cd0 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c06d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c06d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54112/0 (socket says 172.21.15.40:54112) 2023-07-14T11:10:02.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c06d9d700 1 -- 172.21.15.40:0/3059058258 learned_addr learned my addr 172.21.15.40:0/3059058258 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0659c700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 0x7f0c08195dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.765+0000 7f0c0759e700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c08101660 0x7f0c0819ce40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 -- 172.21.15.40:0/3059058258 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 msgr2=0x7f0c08195dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 0x7f0c08195dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 -- 172.21.15.40:0/3059058258 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c08101660 msgr2=0x7f0c0819ce40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c08101660 0x7f0c0819ce40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 -- 172.21.15.40:0/3059058258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bf80095f0 con 0x7f0c080ffae0 2023-07-14T11:10:02.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c06d9d700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f0bf8005740 tx=0x7f0bf8005770 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:02.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf801d070 con 0x7f0c080ffae0 2023-07-14T11:10:02.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0bf8021410 con 0x7f0c080ffae0 2023-07-14T11:10:02.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c0819d960 con 0x7f0c080ffae0 2023-07-14T11:10:02.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0bf8017650 con 0x7f0c080ffae0 2023-07-14T11:10:02.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.766+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c0819dd80 con 0x7f0c080ffae0 2023-07-14T11:10:02.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.767+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c08106a60 con 0x7f0c080ffae0 2023-07-14T11:10:02.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.769+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f0bf8021580 con 0x7f0c080ffae0 2023-07-14T11:10:02.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.771+0000 7f0beffff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 0x7f0bf00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:02.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.771+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f0bf80cdcb0 con 0x7f0c080ffae0 2023-07-14T11:10:02.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.771+0000 7f0c0659c700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 0x7f0bf00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:02.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.771+0000 7f0c0659c700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 0x7f0bf00a3600 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009c60 tx=0x7f0bfc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:02.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.774+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0bf8097240 con 0x7f0c080ffae0 2023-07-14T11:10:02.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.983+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f0c08061120 con 0x7f0c080ffae0 2023-07-14T11:10:02.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.983+0000 7f0beffff700 1 -- 172.21.15.40:0/3059058258 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v88) v1 ==== 93+0+10 (secure 0 0 0) 0x7f0bf809a860 con 0x7f0c080ffae0 2023-07-14T11:10:02.986 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:10:02.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 msgr2=0x7f0bf00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 0x7f0bf00a3600 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009c60 tx=0x7f0bfc009260 comp rx=0 tx=0).stop 2023-07-14T11:10:02.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 msgr2=0x7f0c08195890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:02.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f0bf8005740 tx=0x7f0bf8005770 comp rx=0 tx=0).stop 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 shutdown_connections 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0bf00a1150 0x7f0bf00a3600 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0c08101660 0x7f0c0819ce40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c080ffae0 0x7f0c08195890 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 --2- 172.21.15.40:0/3059058258 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0c08100cd0 0x7f0c08195dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 >> 172.21.15.40:0/3059058258 conn(0x7f0c080fb0e0 msgr2=0x7f0c080fd3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:02.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.985+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 shutdown_connections 2023-07-14T11:10:02.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:02.986+0000 7f0c0d1ff700 1 -- 172.21.15.40:0/3059058258 wait complete. 2023-07-14T11:10:03.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:02 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/76834299' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:03.004 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:02 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/766293525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:03.006 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:10:03.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:02 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/76834299' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:02 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/766293525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:03.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:02 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/76834299' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:03.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:02 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/766293525' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:03.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 -- 172.21.15.40:0/1875907781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200100cd0 msgr2=0x7f9200101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 --2- 172.21.15.40:0/1875907781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200100cd0 0x7f9200101120 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f91f4009990 tx=0x7f91f4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 -- 172.21.15.40:0/1875907781 shutdown_connections 2023-07-14T11:10:03.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 --2- 172.21.15.40:0/1875907781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200101660 0x7f9200107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 --2- 172.21.15.40:0/1875907781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f92000ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 --2- 172.21.15.40:0/1875907781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200100cd0 0x7f9200101120 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 -- 172.21.15.40:0/1875907781 >> 172.21.15.40:0/1875907781 conn(0x7f92000fb0e0 msgr2=0x7f92000fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:03.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 -- 172.21.15.40:0/1875907781 shutdown_connections 2023-07-14T11:10:03.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.210+0000 7f92061ff700 1 -- 172.21.15.40:0/1875907781 wait complete. 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.211+0000 7f92061ff700 1 Processor -- start 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.211+0000 7f92061ff700 1 -- start start 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f9200195850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200100cd0 0x7f9200195d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019d3d0 con 0x7f9200101660 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019d510 con 0x7f92000ffae0 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f92061ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f920019d6a0 con 0x7f9200100cd0 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91fffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f9200195850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91fffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54124/0 (socket says 172.21.15.40:54124) 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91fffff700 1 -- 172.21.15.40:0/1486488880 learned_addr learned my addr 172.21.15.40:0/1486488880 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:03.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91feffd700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200100cd0 0x7f9200195d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 -- 172.21.15.40:0/1486488880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200100cd0 msgr2=0x7f9200195d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200100cd0 0x7f9200195d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 -- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 msgr2=0x7f920019ce00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91ff7fe700 1 -- 172.21.15.40:0/1486488880 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f91f40095f0 con 0x7f92000ffae0 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.212+0000 7f91fffff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f91ff7fe700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f9200195850 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f91f000e9c0 tx=0x7f91f000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:03.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91f0004220 con 0x7f92000ffae0 2023-07-14T11:10:03.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f920019d980 con 0x7f92000ffae0 2023-07-14T11:10:03.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f91f0004380 con 0x7f92000ffae0 2023-07-14T11:10:03.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f91f0003be0 con 0x7f92000ffae0 2023-07-14T11:10:03.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.213+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f920019de90 con 0x7f92000ffae0 2023-07-14T11:10:03.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.214+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f92001070c0 con 0x7f92000ffae0 2023-07-14T11:10:03.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.216+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f91f0003d40 con 0x7f92000ffae0 2023-07-14T11:10:03.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.218+0000 7f91fcff9700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 0x7f91e80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.218+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7f91f0014070 con 0x7f92000ffae0 2023-07-14T11:10:03.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.218+0000 7f91feffd700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 0x7f91e80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.219+0000 7f91feffd700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 0x7f91e80a3600 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f91f400b620 tx=0x7f91f4005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:03.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.220+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f91f0096910 con 0x7f92000ffae0 2023-07-14T11:10:03.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.430+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f920002c9e0 con 0x7f92000ffae0 2023-07-14T11:10:03.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.431+0000 7f91fcff9700 1 -- 172.21.15.40:0/1486488880 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v88) v1 ==== 104+0+11 (secure 0 0 0) 0x7f91f001f340 con 0x7f92000ffae0 2023-07-14T11:10:03.434 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:10:03.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 msgr2=0x7f91e80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 0x7f91e80a3600 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f91f400b620 tx=0x7f91f4005960 comp rx=0 tx=0).stop 2023-07-14T11:10:03.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 msgr2=0x7f9200195850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f9200195850 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f91f000e9c0 tx=0x7f91f000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 shutdown_connections 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f91e80a1150 0x7f91e80a3600 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9200101660 0x7f920019ce00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f92000ffae0 0x7f9200195850 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 --2- 172.21.15.40:0/1486488880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9200100cd0 0x7f9200195d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 >> 172.21.15.40:0/1486488880 conn(0x7f92000fb0e0 msgr2=0x7f9200104200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 shutdown_connections 2023-07-14T11:10:03.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.433+0000 7f92061ff700 1 -- 172.21.15.40:0/1486488880 wait complete. 2023-07-14T11:10:03.454 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:10:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 -- 172.21.15.40:0/1760000799 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350102f40 msgr2=0x7fa350105320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 --2- 172.21.15.40:0/1760000799 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350102f40 0x7fa350105320 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fa338009990 tx=0x7fa338009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 -- 172.21.15.40:0/1760000799 shutdown_connections 2023-07-14T11:10:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 --2- 172.21.15.40:0/1760000799 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa350107c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 --2- 172.21.15.40:0/1760000799 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350100620 0x7fa350102a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 --2- 172.21.15.40:0/1760000799 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350102f40 0x7fa350105320 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 -- 172.21.15.40:0/1760000799 >> 172.21.15.40:0/1760000799 conn(0x7fa3500fa000 msgr2=0x7fa3500fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:03.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 -- 172.21.15.40:0/1760000799 shutdown_connections 2023-07-14T11:10:03.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.661+0000 7fa355fff700 1 -- 172.21.15.40:0/1760000799 wait complete. 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.662+0000 7fa355fff700 1 Processor -- start 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 -- start start 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 0x7fa350193940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa3501a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3501941e0 con 0x7fa350105860 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3501a3b90 con 0x7fa350102f40 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa355fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa3501a3cd0 con 0x7fa350100620 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa34f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa34ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa3501a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa34f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58162/0 (socket says 172.21.15.40:58162) 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa34f7fe700 1 -- 172.21.15.40:0/2129150616 learned_addr learned my addr 172.21.15.40:0/2129150616 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:03.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.663+0000 7fa347fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 0x7fa350193940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 -- 172.21.15.40:0/2129150616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 msgr2=0x7fa350193940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 0x7fa350193940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 -- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 msgr2=0x7fa3501a3650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa3501a3650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 -- 172.21.15.40:0/2129150616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa3380095f0 con 0x7fa350100620 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34ffff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa3501a3650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:03.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa347fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 0x7fa350193940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34f7fe700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fa34000e9a0 tx=0x7fa34000ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa340009790 con 0x7fa350100620 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa3501a3f80 con 0x7fa350100620 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa34000cbd0 con 0x7fa350100620 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa340018680 con 0x7fa350100620 2023-07-14T11:10:03.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.664+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa3501a4400 con 0x7fa350100620 2023-07-14T11:10:03.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.665+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa3501073d0 con 0x7fa350100620 2023-07-14T11:10:03.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.669+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa34000cd40 con 0x7fa350100620 2023-07-14T11:10:03.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.669+0000 7fa34d7fa700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 0x7fa33c0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:03.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.669+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fa340014070 con 0x7fa350100620 2023-07-14T11:10:03.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.669+0000 7fa347fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 0x7fa33c0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:03.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.670+0000 7fa347fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 0x7fa33c0a3680 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fa350194de0 tx=0x7fa338005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:03.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.671+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa340096500 con 0x7fa350100620 2023-07-14T11:10:03.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.885+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fa35002caa0 con 0x7fa350100620 2023-07-14T11:10:03.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.886+0000 7fa34d7fa700 1 -- 172.21.15.40:0/2129150616 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v88) v1 ==== 100+0+11 (secure 0 0 0) 0x7fa3400260a0 con 0x7fa350100620 2023-07-14T11:10:03.888 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:10:03.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 msgr2=0x7fa33c0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 0x7fa33c0a3680 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fa350194de0 tx=0x7fa338005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 msgr2=0x7fa350193400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:03.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fa34000e9a0 tx=0x7fa34000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 shutdown_connections 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa33c0a11d0 0x7fa33c0a3680 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa350105860 0x7fa3501a3650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa350100620 0x7fa350193400 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 --2- 172.21.15.40:0/2129150616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa350102f40 0x7fa350193940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 >> 172.21.15.40:0/2129150616 conn(0x7fa3500fa000 msgr2=0x7fa350101280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:03.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 shutdown_connections 2023-07-14T11:10:03.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:03.888+0000 7fa355fff700 1 -- 172.21.15.40:0/2129150616 wait complete. 2023-07-14T11:10:03.909 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:10:03.970 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:03 smithi040 ceph-mon[97954]: pgmap v286: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:03.970 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:03 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3059058258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:03.970 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:03 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1486488880' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:03 smithi130 ceph-mon[129405]: pgmap v286: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:03 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3059058258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:03 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1486488880' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:03 smithi045 ceph-mon[128791]: pgmap v286: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:03 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3059058258' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:03 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1486488880' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 -- 172.21.15.40:0/3926099061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a0100cd0 msgr2=0x7ff1a0101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/3926099061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0101120 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7ff194009990 tx=0x7ff194009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 -- 172.21.15.40:0/3926099061 shutdown_connections 2023-07-14T11:10:04.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/3926099061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0101660 0x7ff1a0107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/3926099061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/3926099061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0101120 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 -- 172.21.15.40:0/3926099061 >> 172.21.15.40:0/3926099061 conn(0x7ff1a00fb0e0 msgr2=0x7ff1a00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:04.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 -- 172.21.15.40:0/3926099061 shutdown_connections 2023-07-14T11:10:04.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.113+0000 7ff1a4dff700 1 -- 172.21.15.40:0/3926099061 wait complete. 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.114+0000 7ff1a4dff700 1 Processor -- start 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.114+0000 7ff1a4dff700 1 -- start start 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a0101660 0x7ff1a0072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1a00735e0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1a0073750 con 0x7ff1a0100cd0 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff1a4dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff1a019f900 con 0x7ff1a0101660 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19dd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46874/0 (socket says 172.21.15.40:46874) 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 -- 172.21.15.40:0/2012578598 learned_addr learned my addr 172.21.15.40:0/2012578598 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:04.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19ed9d700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a0101660 0x7ff1a0072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 -- 172.21.15.40:0/2012578598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a0101660 msgr2=0x7ff1a0072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a0101660 0x7ff1a0072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 -- 172.21.15.40:0/2012578598 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0100cd0 msgr2=0x7ff1a0072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.115+0000 7ff19e59c700 1 -- 172.21.15.40:0/2012578598 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff1940095f0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff19e59c700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7ff19000d980 tx=0x7ff19000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff19000f890 con 0x7ff1a00ffae0 2023-07-14T11:10:04.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff190010460 con 0x7ff1a00ffae0 2023-07-14T11:10:04.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff1a019fbe0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff1900195c0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.116+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff1a01a00f0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.117+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff1a01070c0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.119+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7ff1900105d0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.121+0000 7ff18f7fe700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 0x7ff1880a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.121+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7ff1900cc8c0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.121+0000 7ff19dd9b700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 0x7ff1880a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.121+0000 7ff19dd9b700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 0x7ff1880a3600 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff19400b5c0 tx=0x7ff194005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:04.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.125+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff190095dc0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.335+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7ff1a00611f0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.335+0000 7ff18f7fe700 1 -- 172.21.15.40:0/2012578598 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v88) v1 ==== 103+0+11 (secure 0 0 0) 0x7ff1900200a0 con 0x7ff1a00ffae0 2023-07-14T11:10:04.338 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 msgr2=0x7ff1880a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 0x7ff1880a3600 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7ff19400b5c0 tx=0x7ff194005fd0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 msgr2=0x7ff1a00742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7ff19000d980 tx=0x7ff19000dc90 comp rx=0 tx=0).stop 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 shutdown_connections 2023-07-14T11:10:04.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff1880a1150 0x7ff1880a3600 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff1a0101660 0x7ff1a0072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff1a00ffae0 0x7ff1a00742c0 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 --2- 172.21.15.40:0/2012578598 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff1a0100cd0 0x7ff1a0072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 >> 172.21.15.40:0/2012578598 conn(0x7ff1a00fb0e0 msgr2=0x7ff1a0104200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 shutdown_connections 2023-07-14T11:10:04.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.338+0000 7ff1a4dff700 1 -- 172.21.15.40:0/2012578598 wait complete. 2023-07-14T11:10:04.360 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2023-07-14T11:10:04.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 -- 172.21.15.40:0/4129161395 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438074550 msgr2=0x7fb4380729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 --2- 172.21.15.40:0/4129161395 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438074550 0x7fb4380729a0 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fb42c009990 tx=0x7fb42c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 -- 172.21.15.40:0/4129161395 shutdown_connections 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 --2- 172.21.15.40:0/4129161395 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 0x7fb438109e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 --2- 172.21.15.40:0/4129161395 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438074550 0x7fb4380729a0 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 --2- 172.21.15.40:0/4129161395 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438072f70 0x7fb4380733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 -- 172.21.15.40:0/4129161395 >> 172.21.15.40:0/4129161395 conn(0x7fb4380fb0e0 msgr2=0x7fb4380fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 -- 172.21.15.40:0/4129161395 shutdown_connections 2023-07-14T11:10:04.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.561+0000 7fb43edff700 1 -- 172.21.15.40:0/4129161395 wait complete. 2023-07-14T11:10:04.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.562+0000 7fb43edff700 1 Processor -- start 2023-07-14T11:10:04.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 -- start start 2023-07-14T11:10:04.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438074550 0x7fb438197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 0x7fb43819f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb43819f630 con 0x7fb438072f70 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb43819f770 con 0x7fb438074550 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43edff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb43819f900 con 0x7fb4381079a0 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43cb9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43cb9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46892/0 (socket says 172.21.15.40:46892) 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43cb9b700 1 -- 172.21.15.40:0/1472730014 learned_addr learned my addr 172.21.15.40:0/1472730014 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:04.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb43d39c700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 0x7fb43819f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.563+0000 7fb437fff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438074550 0x7fb438197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 -- 172.21.15.40:0/1472730014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 msgr2=0x7fb43819f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 0x7fb43819f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 -- 172.21.15.40:0/1472730014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438074550 msgr2=0x7fb438197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438074550 0x7fb438197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 -- 172.21.15.40:0/1472730014 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb42c0095f0 con 0x7fb438072f70 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43cb9b700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fb42c005720 tx=0x7fb42c009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:04.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42c01d070 con 0x7fb438072f70 2023-07-14T11:10:04.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb42c021410 con 0x7fb438072f70 2023-07-14T11:10:04.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb42c017650 con 0x7fb438072f70 2023-07-14T11:10:04.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb43819fb80 con 0x7fb438072f70 2023-07-14T11:10:04.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.564+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb43819ff50 con 0x7fb438072f70 2023-07-14T11:10:04.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.565+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb438106050 con 0x7fb438072f70 2023-07-14T11:10:04.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.568+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb42c021580 con 0x7fb438072f70 2023-07-14T11:10:04.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.570+0000 7fb435ffb700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 0x7fb4280a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:04.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.570+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(88..88 src has 1..88) v4 ==== 8730+0+0 (secure 0 0 0) 0x7fb42c0ce0b0 con 0x7fb438072f70 2023-07-14T11:10:04.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.570+0000 7fb437fff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 0x7fb4280a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:04.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.570+0000 7fb437fff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 0x7fb4280a3680 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fb438074270 tx=0x7fb424009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:04.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.571+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb42c097640 con 0x7fb438072f70 2023-07-14T11:10:04.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.798+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7fb43802c850 con 0x7fb438072f70 2023-07-14T11:10:04.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.799+0000 7fb435ffb700 1 -- 172.21.15.40:0/1472730014 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 10 entries v88) v1 ==== 81+0+589 (secure 0 0 0) 0x7fb42c09ac60 con 0x7fb438072f70 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3160212490 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/4083556243 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/2326174215 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/1897779716 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:04.801 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3378800182 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:04.802 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:04.802 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:04.802 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:10:04.802 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/500170929 2023-07-15T11:02:39.441675+0000 2023-07-14T11:10:04.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 msgr2=0x7fb4280a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 0x7fb4280a3680 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fb438074270 tx=0x7fb424009260 comp rx=0 tx=0).stop 2023-07-14T11:10:04.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 msgr2=0x7fb438197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:04.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fb42c005720 tx=0x7fb42c009580 comp rx=0 tx=0).stop 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 shutdown_connections 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4280a11d0 0x7fb4280a3680 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4381079a0 0x7fb43819f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb438072f70 0x7fb438197ab0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 --2- 172.21.15.40:0/1472730014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb438074550 0x7fb438197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.801+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 >> 172.21.15.40:0/1472730014 conn(0x7fb4380fb0e0 msgr2=0x7fb4380fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:04.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.802+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 shutdown_connections 2023-07-14T11:10:04.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:04.802+0000 7fb43edff700 1 -- 172.21.15.40:0/1472730014 wait complete. 2023-07-14T11:10:04.805 INFO:teuthology.orchestra.run.smithi040.stderr:listed 10 entries 2023-07-14T11:10:04.821 INFO:tasks.cephfs.fuse_mount:Running fusermount -u on ubuntu@smithi040.front.sepia.ceph.com... 2023-07-14T11:10:04.821 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:10:04.822 DEBUG:teuthology.orchestra.run.smithi040:> sudo fusermount -u /home/ubuntu/cephtest/mnt.admin 2023-07-14T11:10:04.892 INFO:tasks.cephfs.fuse_mount.ceph-fuse.admin.smithi040.stderr:ceph-fuse[134663]: fuse finished with error 0 and tester_r 0 2023-07-14T11:10:04.895 INFO:teuthology.orchestra.run:waiting for 300 2023-07-14T11:10:04.997 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:04 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2129150616' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:04.997 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:04 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2012578598' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:05.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:04 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2129150616' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:04 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2012578598' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:05.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:04 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2129150616' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:10:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:04 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2012578598' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:10:06.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:05 smithi130 ceph-mon[129405]: pgmap v287: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:05 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1472730014' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:06.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:05 smithi040 ceph-mon[97954]: pgmap v287: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:05 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1472730014' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:05 smithi045 ceph-mon[128791]: pgmap v287: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:05 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1472730014' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]: dispatch 2023-07-14T11:10:07.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]: dispatch 2023-07-14T11:10:07.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]: dispatch 2023-07-14T11:10:07.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]: dispatch 2023-07-14T11:10:07.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]: dispatch 2023-07-14T11:10:07.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:06 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]: dispatch 2023-07-14T11:10:07.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]: dispatch 2023-07-14T11:10:07.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]: dispatch 2023-07-14T11:10:07.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]: dispatch 2023-07-14T11:10:07.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:06 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]: dispatch 2023-07-14T11:10:07.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]: dispatch 2023-07-14T11:10:07.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]: dispatch 2023-07-14T11:10:07.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]: dispatch 2023-07-14T11:10:07.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:06 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]: dispatch 2023-07-14T11:10:08.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: pgmap v288: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]': finished 2023-07-14T11:10:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]': finished 2023-07-14T11:10:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]': finished 2023-07-14T11:10:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]': finished 2023-07-14T11:10:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]': finished 2023-07-14T11:10:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]': finished 2023-07-14T11:10:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:07 smithi130 ceph-mon[129405]: osdmap e89: 12 total, 12 up, 12 in 2023-07-14T11:10:08.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: pgmap v288: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]': finished 2023-07-14T11:10:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]': finished 2023-07-14T11:10:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]': finished 2023-07-14T11:10:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]': finished 2023-07-14T11:10:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]': finished 2023-07-14T11:10:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:07 smithi040 ceph-mon[97954]: osdmap e89: 12 total, 12 up, 12 in 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: pgmap v288: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 341 B/s wr, 0 op/s 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.0", "id": [8, 1]}]': finished 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.6", "id": [8, 11]}]': finished 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.8", "id": [2, 11]}]': finished 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.e", "id": [2, 4]}]': finished 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 6]}]': finished 2023-07-14T11:10:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 8]}]': finished 2023-07-14T11:10:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.24", "id": [4, 9]}]': finished 2023-07-14T11:10:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.25", "id": [5, 9]}]': finished 2023-07-14T11:10:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.27", "id": [3, 9]}]': finished 2023-07-14T11:10:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [9, 8]}]': finished 2023-07-14T11:10:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:07 smithi045 ceph-mon[128791]: osdmap e89: 12 total, 12 up, 12 in 2023-07-14T11:10:09.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:08 smithi130 ceph-mon[129405]: osdmap e90: 12 total, 12 up, 12 in 2023-07-14T11:10:09.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:08 smithi040 ceph-mon[97954]: osdmap e90: 12 total, 12 up, 12 in 2023-07-14T11:10:09.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:08 smithi045 ceph-mon[128791]: osdmap e90: 12 total, 12 up, 12 in 2023-07-14T11:10:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:09 smithi130 ceph-mon[129405]: pgmap v291: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:10:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:09 smithi040 ceph-mon[97954]: pgmap v291: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:10:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:09 smithi045 ceph-mon[128791]: pgmap v291: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:10:10.997 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:10:10.997 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:10:10.998 DEBUG:teuthology.orchestra.run.smithi040:> (cd /home/ubuntu/cephtest && exec rmdir -- /home/ubuntu/cephtest/mnt.admin) 2023-07-14T11:10:11.063 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/563075378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf34074550 msgr2=0x7fcf340729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/563075378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf34074550 0x7fcf340729a0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fcf240098f0 tx=0x7fcf24009c00 comp rx=0 tx=0).stop 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/563075378 shutdown_connections 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/563075378 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf3410c0d0 0x7fcf3410e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/563075378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf34074550 0x7fcf340729a0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/563075378 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 0x7fcf340733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/563075378 >> 172.21.15.40:0/563075378 conn(0x7fcf340fb0e0 msgr2=0x7fcf340fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:11.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/563075378 shutdown_connections 2023-07-14T11:10:11.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.271+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/563075378 wait complete. 2023-07-14T11:10:11.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 Processor -- start 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 -- start start 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 0x7fcf3419be90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 0x7fcf3419c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf341a3a10 con 0x7fcf3410c0d0 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf341a3b50 con 0x7fcf34072f70 2023-07-14T11:10:11.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.272+0000 7fcf3a7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcf341a3ce0 con 0x7fcf34074550 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46916/0 (socket says 172.21.15.40:46916) 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 -- 172.21.15.40:0/2274412152 learned_addr learned my addr 172.21.15.40:0/2274412152 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf33fff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 0x7fcf3419be90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf337fe700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 0x7fcf3419c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 -- 172.21.15.40:0/2274412152 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 msgr2=0x7fcf3419c3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 0x7fcf3419c3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 -- 172.21.15.40:0/2274412152 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 msgr2=0x7fcf3419be90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 0x7fcf3419be90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 -- 172.21.15.40:0/2274412152 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcf280095f0 con 0x7fcf3410c0d0 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf337fe700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 0x7fcf3419c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.273+0000 7fcf38d9c700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fcf2c00ef30 tx=0x7fcf2c00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.274+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf2c018070 con 0x7fcf3410c0d0 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.274+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcf2c00f040 con 0x7fcf3410c0d0 2023-07-14T11:10:11.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.274+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcf24009520 con 0x7fcf3410c0d0 2023-07-14T11:10:11.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.274+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcf2c00cd00 con 0x7fcf3410c0d0 2023-07-14T11:10:11.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.274+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcf341a4280 con 0x7fcf3410c0d0 2023-07-14T11:10:11.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.275+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcf34101c20 con 0x7fcf3410c0d0 2023-07-14T11:10:11.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.277+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fcf2c003a00 con 0x7fcf3410c0d0 2023-07-14T11:10:11.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.278+0000 7fcf317fa700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 0x7fcf1c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.278+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fcf2c0cc760 con 0x7fcf3410c0d0 2023-07-14T11:10:11.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.279+0000 7fcf33fff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 0x7fcf1c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.279+0000 7fcf33fff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 0x7fcf1c0a3650 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fcf24009fd0 tx=0x7fcf24019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:11.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.282+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcf2c095bd0 con 0x7fcf3410c0d0 2023-07-14T11:10:11.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.491+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcf3402c850 con 0x7fcf3410c0d0 2023-07-14T11:10:11.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.493+0000 7fcf317fa700 1 -- 172.21.15.40:0/2274412152 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+24369 (secure 0 0 0) 0x7fcf2c0991f0 con 0x7fcf3410c0d0 2023-07-14T11:10:11.495 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:11.495 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":90,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:10:07.729799+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 msgr2=0x7fcf1c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 0x7fcf1c0a3650 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fcf24009fd0 tx=0x7fcf24019040 comp rx=0 tx=0).stop 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 msgr2=0x7fcf341a3440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fcf2c00ef30 tx=0x7fcf2c00c3a0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 shutdown_connections 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcf1c0a11a0 0x7fcf1c0a3650 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcf3410c0d0 0x7fcf341a3440 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcf34072f70 0x7fcf3419be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 --2- 172.21.15.40:0/2274412152 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcf34074550 0x7fcf3419c3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 >> 172.21.15.40:0/2274412152 conn(0x7fcf340fb0e0 msgr2=0x7fcf340fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:11.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 shutdown_connections 2023-07-14T11:10:11.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.495+0000 7fcf3a7ff700 1 -- 172.21.15.40:0/2274412152 wait complete. 2023-07-14T11:10:11.515 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:10:11.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 -- 172.21.15.40:0/2296232337 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e40ffb00 msgr2=0x7f44e40fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/2296232337 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e40ffb00 0x7f44e40fff10 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f44e00098f0 tx=0x7f44e0009c00 comp rx=0 tx=0).stop 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 -- 172.21.15.40:0/2296232337 shutdown_connections 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/2296232337 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e4101680 0x7f44e4107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/2296232337 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e40ffb00 0x7f44e40fff10 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/2296232337 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 0x7f44e4101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 -- 172.21.15.40:0/2296232337 >> 172.21.15.40:0/2296232337 conn(0x7f44e40fb0e0 msgr2=0x7f44e40fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:11.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 -- 172.21.15.40:0/2296232337 shutdown_connections 2023-07-14T11:10:11.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.721+0000 7f44ec1ff700 1 -- 172.21.15.40:0/2296232337 wait complete. 2023-07-14T11:10:11.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 Processor -- start 2023-07-14T11:10:11.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 -- start start 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e40ffb00 0x7f44e4197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 0x7f44e4197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44e419f5f0 con 0x7f44e4101680 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44e419f730 con 0x7f44e40ffb00 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.722+0000 7f44ec1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44e419f8c0 con 0x7f44e4100cf0 2023-07-14T11:10:11.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46946/0 (socket says 172.21.15.40:46946) 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 -- 172.21.15.40:0/547747084 learned_addr learned my addr 172.21.15.40:0/547747084 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44e9f9b700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e40ffb00 0x7f44e4197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44e979a700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 0x7f44e4197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 -- 172.21.15.40:0/547747084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 msgr2=0x7f44e4197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 0x7f44e4197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 -- 172.21.15.40:0/547747084 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e40ffb00 msgr2=0x7f44e4197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e40ffb00 0x7f44e4197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 -- 172.21.15.40:0/547747084 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44e0009520 con 0x7f44e4101680 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44ea79c700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f44dc00ef30 tx=0x7f44dc00c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.723+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44dc018070 con 0x7f44e4101680 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.724+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44dc00f040 con 0x7f44e4101680 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.724+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44e419fb40 con 0x7f44e4101680 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.724+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44dc00ccf0 con 0x7f44e4101680 2023-07-14T11:10:11.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.724+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44e41a0050 con 0x7f44e4101680 2023-07-14T11:10:11.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.725+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f44e41921e0 con 0x7f44e4101680 2023-07-14T11:10:11.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.727+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f44dc003a00 con 0x7f44e4101680 2023-07-14T11:10:11.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.728+0000 7f44daffd700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 0x7f44d00a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:11.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.728+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f44dc0cc760 con 0x7f44e4101680 2023-07-14T11:10:11.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.728+0000 7f44e9f9b700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 0x7f44d00a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:11.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.729+0000 7f44e9f9b700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 0x7f44d00a3650 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f44e0009fd0 tx=0x7f44e0019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:11.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.732+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f44dc095bd0 con 0x7f44e4101680 2023-07-14T11:10:11.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:11 smithi040 ceph-mon[97954]: pgmap v292: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:11.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2274412152' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:11.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.944+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f44e402c850 con 0x7f44e4101680 2023-07-14T11:10:11.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.946+0000 7f44daffd700 1 -- 172.21.15.40:0/547747084 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+24369 (secure 0 0 0) 0x7f44dc0991f0 con 0x7f44e4101680 2023-07-14T11:10:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:11.948 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":90,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:10:07.729799+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:11.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 msgr2=0x7f44d00a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 0x7f44d00a3650 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f44e0009fd0 tx=0x7f44e0019040 comp rx=0 tx=0).stop 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 msgr2=0x7f44e419f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f44dc00ef30 tx=0x7f44dc00c3a0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 shutdown_connections 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44d00a11a0 0x7f44d00a3650 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44e4101680 0x7f44e419f020 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44e40ffb00 0x7f44e4197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 --2- 172.21.15.40:0/547747084 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44e4100cf0 0x7f44e4197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:11.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 >> 172.21.15.40:0/547747084 conn(0x7f44e40fb0e0 msgr2=0x7f44e41065f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:11.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 shutdown_connections 2023-07-14T11:10:11.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:11.949+0000 7f44ec1ff700 1 -- 172.21.15.40:0/547747084 wait complete. 2023-07-14T11:10:11.969 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs standby_count_wanted 0 2023-07-14T11:10:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:11 smithi130 ceph-mon[129405]: pgmap v292: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2274412152' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:11 smithi045 ceph-mon[128791]: pgmap v292: 129 pgs: 3 peering, 126 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2274412152' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 -- 172.21.15.40:0/1497065985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 msgr2=0x7fe4280ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 --2- 172.21.15.40:0/1497065985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe4280ffef0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fe424009990 tx=0x7fe424009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 -- 172.21.15.40:0/1497065985 shutdown_connections 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 --2- 172.21.15.40:0/1497065985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe428107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 --2- 172.21.15.40:0/1497065985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe4280ffef0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 --2- 172.21.15.40:0/1497065985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 -- 172.21.15.40:0/1497065985 >> 172.21.15.40:0/1497065985 conn(0x7fe4280fb0e0 msgr2=0x7fe4280fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:12.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 -- 172.21.15.40:0/1497065985 shutdown_connections 2023-07-14T11:10:12.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.177+0000 7fe42ffff700 1 -- 172.21.15.40:0/1497065985 wait complete. 2023-07-14T11:10:12.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.178+0000 7fe42ffff700 1 Processor -- start 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.178+0000 7fe42ffff700 1 -- start start 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.178+0000 7fe42ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42ffff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe42819f580 con 0x7fe428100cd0 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42ffff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe42819f6c0 con 0x7fe428101660 2023-07-14T11:10:12.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42ffff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe42819f850 con 0x7fe4280ffae0 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42d59a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58260/0 (socket says 172.21.15.40:58260) 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42d59a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46972/0 (socket says 172.21.15.40:46972) 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:52092/0 (socket says 172.21.15.40:52092) 2023-07-14T11:10:12.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 -- 172.21.15.40:0/974691840 learned_addr learned my addr 172.21.15.40:0/974691840 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 -- 172.21.15.40:0/974691840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 msgr2=0x7fe42819efb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 -- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 msgr2=0x7fe428197f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42dd9b700 1 -- 172.21.15.40:0/974691840 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe418009520 con 0x7fe4280ffae0 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42e59c700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.179+0000 7fe42d59a700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:12.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe42dd9b700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe424000c00 tx=0x7fe4240057a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:12.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe42401d070 con 0x7fe4280ffae0 2023-07-14T11:10:12.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4240095f0 con 0x7fe4280ffae0 2023-07-14T11:10:12.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe42400fbc0 con 0x7fe4280ffae0 2023-07-14T11:10:12.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4240173f0 con 0x7fe4280ffae0 2023-07-14T11:10:12.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.180+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe42819fe50 con 0x7fe4280ffae0 2023-07-14T11:10:12.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.181+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe428192110 con 0x7fe4280ffae0 2023-07-14T11:10:12.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.183+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fe424021410 con 0x7fe4280ffae0 2023-07-14T11:10:12.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.184+0000 7fe41effd700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 0x7fe4140a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.185+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fe4240ce660 con 0x7fe4280ffae0 2023-07-14T11:10:12.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.185+0000 7fe42d59a700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 0x7fe4140a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.185+0000 7fe42d59a700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 0x7fe4140a3600 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fe418009fd0 tx=0x7fe418009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:12.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.186+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fe424097ad0 con 0x7fe4280ffae0 2023-07-14T11:10:12.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.442+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"} v 0) v1 -- 0x7fe42802c850 con 0x7fe4280ffae0 2023-07-14T11:10:12.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.762+0000 7fe41effd700 1 -- 172.21.15.40:0/974691840 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]=0 v17) v1 ==== 118+0+0 (secure 0 0 0) 0x7fe42409b0f0 con 0x7fe4280ffae0 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 msgr2=0x7fe4140a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 0x7fe4140a3600 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fe418009fd0 tx=0x7fe418009260 comp rx=0 tx=0).stop 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 msgr2=0x7fe428197a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fe424000c00 tx=0x7fe4240057a0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 shutdown_connections 2023-07-14T11:10:12.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4140a1150 0x7fe4140a3600 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe428101660 0x7fe42819efb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4280ffae0 0x7fe428197a00 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 --2- 172.21.15.40:0/974691840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe428100cd0 0x7fe428197f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 >> 172.21.15.40:0/974691840 conn(0x7fe4280fb0e0 msgr2=0x7fe428106600 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 shutdown_connections 2023-07-14T11:10:12.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.765+0000 7fe42ffff700 1 -- 172.21.15.40:0/974691840 wait complete. 2023-07-14T11:10:12.786 DEBUG:tasks.cephfs.filesystem:fs_config: {'ec_profile': ['m=2', 'k=2', 'crush-failure-domain=osd'], 'max_mds': 5, 'session_timeout': 300, 'standby_replay': True} 2023-07-14T11:10:12.787 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs max_mds 5 2023-07-14T11:10:12.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 -- 172.21.15.40:0/1273253652 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 msgr2=0x7f4308101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 --2- 172.21.15.40:0/1273253652 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308101120 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f42fc009990 tx=0x7f42fc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 -- 172.21.15.40:0/1273253652 shutdown_connections 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 --2- 172.21.15.40:0/1273253652 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4308101660 0x7f4308107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 --2- 172.21.15.40:0/1273253652 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f43080ffae0 0x7f43080ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 --2- 172.21.15.40:0/1273253652 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308101120 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 -- 172.21.15.40:0/1273253652 >> 172.21.15.40:0/1273253652 conn(0x7f43080fb0e0 msgr2=0x7f43080fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:12.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 -- 172.21.15.40:0/1273253652 shutdown_connections 2023-07-14T11:10:12.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.993+0000 7f430d3ff700 1 -- 172.21.15.40:0/1273253652 wait complete. 2023-07-14T11:10:12.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 Processor -- start 2023-07-14T11:10:12.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 -- start start 2023-07-14T11:10:12.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f43080ffae0 0x7f43080742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 0x7f4308072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f43080735e0 con 0x7f4308100cd0 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4308073750 con 0x7f43080ffae0 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f430d3ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f430819f900 con 0x7f4308101660 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.994+0000 7f43067fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46982/0 (socket says 172.21.15.40:46982) 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 -- 172.21.15.40:0/3642913167 learned_addr learned my addr 172.21.15.40:0/3642913167 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f4306ffd700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f43080ffae0 0x7f43080742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43077fe700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 0x7f4308072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 -- 172.21.15.40:0/3642913167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 msgr2=0x7f4308072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 0x7f4308072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 -- 172.21.15.40:0/3642913167 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f43080ffae0 msgr2=0x7f43080742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f43080ffae0 0x7f43080742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:12.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 -- 172.21.15.40:0/3642913167 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4300009c40 con 0x7f4308100cd0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43077fe700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 0x7f4308072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f43067fc700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f42fc004760 tx=0x7f42fc004790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42fc01d070 con 0x7f4308100cd0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f42fc004830 con 0x7f4308100cd0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42fc005480 con 0x7f4308100cd0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.995+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42fc0095f0 con 0x7f4308100cd0 2023-07-14T11:10:13.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.996+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f430819fd60 con 0x7f4308100cd0 2023-07-14T11:10:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.996+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f43081070c0 con 0x7f4308100cd0 2023-07-14T11:10:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:12.999+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f42fc00fad0 con 0x7f4308100cd0 2023-07-14T11:10:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.000+0000 7f42effff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 0x7f42f00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.000+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f42fc0cdfe0 con 0x7f4308100cd0 2023-07-14T11:10:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.000+0000 7f4306ffd700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 0x7f42f00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.001+0000 7f4306ffd700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 0x7f42f00a3600 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f4308100b30 tx=0x7f42f8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:13.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.002+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f42fc097490 con 0x7f4308100cd0 2023-07-14T11:10:13.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:12 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/547747084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:13.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:12 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/974691840' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:12 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:12 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/547747084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:13.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:12 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/974691840' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:12 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:12 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/547747084' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:12 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/974691840' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:12 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2023-07-14T11:10:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.258+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"} v 0) v1 -- 0x7f430802caa0 con 0x7f4308100cd0 2023-07-14T11:10:13.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.767+0000 7f42effff700 1 -- 172.21.15.40:0/3642913167 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]=0 v18) v1 ==== 105+0+0 (secure 0 0 0) 0x7f42fc09aab0 con 0x7f4308100cd0 2023-07-14T11:10:13.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 msgr2=0x7f42f00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 0x7f42f00a3600 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f4308100b30 tx=0x7f42f8009260 comp rx=0 tx=0).stop 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 msgr2=0x7f4308072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f42fc004760 tx=0x7f42fc004790 comp rx=0 tx=0).stop 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 shutdown_connections 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f42f00a1150 0x7f42f00a3600 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4308101660 0x7f4308072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:13.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f43080ffae0 0x7f43080742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:13.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 --2- 172.21.15.40:0/3642913167 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4308100cd0 0x7f4308072900 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:13.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 >> 172.21.15.40:0/3642913167 conn(0x7f43080fb0e0 msgr2=0x7f4308104200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:13.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 shutdown_connections 2023-07-14T11:10:13.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:13.770+0000 7f430d3ff700 1 -- 172.21.15.40:0/3642913167 wait complete. 2023-07-14T11:10:13.796 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs allow_standby_replay true 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 -- 172.21.15.40:0/3632731067 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc0ffae0 msgr2=0x7f9efc0ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 --2- 172.21.15.40:0/3632731067 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc0ffae0 0x7f9efc0ffef0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f9ef8009990 tx=0x7f9ef8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 -- 172.21.15.40:0/3632731067 shutdown_connections 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 --2- 172.21.15.40:0/3632731067 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc101660 0x7f9efc107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 --2- 172.21.15.40:0/3632731067 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc0ffae0 0x7f9efc0ffef0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 --2- 172.21.15.40:0/3632731067 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc100cd0 0x7f9efc101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 -- 172.21.15.40:0/3632731067 >> 172.21.15.40:0/3632731067 conn(0x7f9efc0fb0e0 msgr2=0x7f9efc0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 -- 172.21.15.40:0/3632731067 shutdown_connections 2023-07-14T11:10:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.002+0000 7f9f041ff700 1 -- 172.21.15.40:0/3632731067 wait complete. 2023-07-14T11:10:14.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.003+0000 7f9f041ff700 1 Processor -- start 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.003+0000 7f9f041ff700 1 -- start start 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc0ffae0 0x7f9efc197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 0x7f9efc197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9efc19f5f0 con 0x7f9efc101660 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9efc19f730 con 0x7f9efc100cd0 2023-07-14T11:10:14.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f041ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9efc19f8c0 con 0x7f9efc0ffae0 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0179a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 0x7f9efc197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0179a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 0x7f9efc197fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38496/0 (socket says 172.21.15.40:38496) 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:43036/0 (socket says 172.21.15.40:43036) 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0179a700 1 -- 172.21.15.40:0/4282859807 learned_addr learned my addr 172.21.15.40:0/4282859807 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f01f9b700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc0ffae0 0x7f9efc197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 -- 172.21.15.40:0/4282859807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc0ffae0 msgr2=0x7f9efc197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc0ffae0 0x7f9efc197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 -- 172.21.15.40:0/4282859807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 msgr2=0x7f9efc197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 0x7f9efc197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.004+0000 7f9f0279c700 1 -- 172.21.15.40:0/4282859807 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ef80095f0 con 0x7f9efc101660 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9f0279c700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f9ef400efc0 tx=0x7f9ef400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ef4018070 con 0x7f9efc101660 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9efc19fb40 con 0x7f9efc101660 2023-07-14T11:10:14.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9ef400f040 con 0x7f9efc101660 2023-07-14T11:10:14.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9ef400cd40 con 0x7f9efc101660 2023-07-14T11:10:14.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.005+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9efc1a00a0 con 0x7f9efc101660 2023-07-14T11:10:14.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.006+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9efc1921c0 con 0x7f9efc101660 2023-07-14T11:10:14.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.008+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9ef4003a00 con 0x7f9efc101660 2023-07-14T11:10:14.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.009+0000 7f9ef2ffd700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 0x7f9ee80a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.010+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f9ef40cd130 con 0x7f9efc101660 2023-07-14T11:10:14.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.010+0000 7f9f01f9b700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 0x7f9ee80a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.010+0000 7f9f01f9b700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 0x7f9ee80a3680 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f9ef800b2e0 tx=0x7f9ef8005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:14.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.012+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9ef40965a0 con 0x7f9efc101660 2023-07-14T11:10:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:13 smithi130 ceph-mon[129405]: pgmap v293: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:13 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2023-07-14T11:10:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:13 smithi130 ceph-mon[129405]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:13 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2023-07-14T11:10:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:13 smithi040 ceph-mon[97954]: pgmap v293: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:13 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2023-07-14T11:10:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:13 smithi040 ceph-mon[97954]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:13 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2023-07-14T11:10:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:13 smithi045 ceph-mon[128791]: pgmap v293: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 127 B/s wr, 0 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:13 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2023-07-14T11:10:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:13 smithi045 ceph-mon[128791]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:13 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]: dispatch 2023-07-14T11:10:14.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.268+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) v1 -- 0x7f9efc02c850 con 0x7f9efc101660 2023-07-14T11:10:14.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.345+0000 7f9ef2ffd700 1 -- 172.21.15.40:0/4282859807 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v20) v1 ==== 121+0+0 (secure 0 0 0) 0x7f9ef4099bc0 con 0x7f9efc101660 2023-07-14T11:10:14.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.348+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 msgr2=0x7f9ee80a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 0x7f9ee80a3680 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f9ef800b2e0 tx=0x7f9ef8005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 msgr2=0x7f9efc19f020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f9ef400efc0 tx=0x7f9ef400c3a0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 shutdown_connections 2023-07-14T11:10:14.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ee80a11d0 0x7f9ee80a3680 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9efc101660 0x7f9efc19f020 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9efc0ffae0 0x7f9efc197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 --2- 172.21.15.40:0/4282859807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9efc100cd0 0x7f9efc197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 >> 172.21.15.40:0/4282859807 conn(0x7f9efc0fb0e0 msgr2=0x7f9efc1065a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:14.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 shutdown_connections 2023-07-14T11:10:14.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.349+0000 7f9f041ff700 1 -- 172.21.15.40:0/4282859807 wait complete. 2023-07-14T11:10:14.373 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set cephfs session_timeout 300 2023-07-14T11:10:14.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 -- 172.21.15.40:0/1013437010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 msgr2=0x7f181c0ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 --2- 172.21.15.40:0/1013437010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0ffef0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f180c009990 tx=0x7f180c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 -- 172.21.15.40:0/1013437010 shutdown_connections 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 --2- 172.21.15.40:0/1013437010 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c101660 0x7f181c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 --2- 172.21.15.40:0/1013437010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0ffef0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 --2- 172.21.15.40:0/1013437010 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c100cd0 0x7f181c101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 -- 172.21.15.40:0/1013437010 >> 172.21.15.40:0/1013437010 conn(0x7f181c0fb0e0 msgr2=0x7f181c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 -- 172.21.15.40:0/1013437010 shutdown_connections 2023-07-14T11:10:14.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.579+0000 7f18225ff700 1 -- 172.21.15.40:0/1013437010 wait complete. 2023-07-14T11:10:14.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.580+0000 7f18225ff700 1 Processor -- start 2023-07-14T11:10:14.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 -- start start 2023-07-14T11:10:14.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 0x7f181c072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 0x7f181c072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181c0735e0 con 0x7f181c101660 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181c073750 con 0x7f181c100cd0 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f18225ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f181c19f8d0 con 0x7f181c0ffae0 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f181bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f1820b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 0x7f181c072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f181bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59446/0 (socket says 172.21.15.40:59446) 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f181bfff700 1 -- 172.21.15.40:0/2049151387 learned_addr learned my addr 172.21.15.40:0/2049151387 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:14.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.581+0000 7f1813fff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 0x7f181c072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 -- 172.21.15.40:0/2049151387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 msgr2=0x7f181c0742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 -- 172.21.15.40:0/2049151387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 msgr2=0x7f181c072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 0x7f181c072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 -- 172.21.15.40:0/2049151387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f180c0095f0 con 0x7f181c101660 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1813fff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 0x7f181c072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:14.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1820b9c700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 0x7f181c072e40 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f1814007f10 tx=0x7f181400d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1814009ea0 con 0x7f181c101660 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f181c19fbb0 con 0x7f181c101660 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f181400f040 con 0x7f181c101660 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1814004400 con 0x7f181c101660 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.582+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f181c1a0110 con 0x7f181c101660 2023-07-14T11:10:14.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.583+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f181c107120 con 0x7f181c101660 2023-07-14T11:10:14.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.586+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1814004ad0 con 0x7f181c101660 2023-07-14T11:10:14.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.587+0000 7f1819ffb700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 0x7f18080a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:14.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.587+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f18140cce40 con 0x7f181c101660 2023-07-14T11:10:14.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.588+0000 7f181bfff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 0x7f18080a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:14.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.588+0000 7f181bfff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 0x7f18080a3680 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f180c006010 tx=0x7f180c005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:14.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.590+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1814096220 con 0x7f181c101660 2023-07-14T11:10:14.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:14.853+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"} v 0) v1 -- 0x7f181c02c850 con 0x7f181c101660 2023-07-14T11:10:15.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: daemon mds.a assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: fsmap cephfs:2 {0=b=up:active,1=a=up:creating} 10 up:standby 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: daemon mds.a is now active in filesystem cephfs as rank 1 2023-07-14T11:10:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: fsmap cephfs:2 {0=b=up:active,1=a=up:active} 2 up:standby-replay 8 up:standby 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:creating} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:14 smithi130 ceph-mon[129405]: daemon mds.c is now active in filesystem cephfs as rank 2 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: daemon mds.a assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: fsmap cephfs:2 {0=b=up:active,1=a=up:creating} 10 up:standby 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: daemon mds.a is now active in filesystem cephfs as rank 1 2023-07-14T11:10:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: fsmap cephfs:2 {0=b=up:active,1=a=up:active} 2 up:standby-replay 8 up:standby 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:creating} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:14 smithi040 ceph-mon[97954]: daemon mds.c is now active in filesystem cephfs as rank 2 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3642913167' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "5"}]': finished 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: daemon mds.a assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: fsmap cephfs:1 {0=b=up:active} 11 up:standby 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: fsmap cephfs:2 {0=b=up:active,1=a=up:creating} 10 up:standby 2023-07-14T11:10:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: daemon mds.a is now active in filesystem cephfs as rank 1 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4282859807' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: daemon mds.c assigned to filesystem cephfs as rank 2 (now has 3 ranks) 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: fsmap cephfs:2 {0=b=up:active,1=a=up:active} 2 up:standby-replay 8 up:standby 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:creating} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:14 smithi045 ceph-mon[128791]: daemon mds.c is now active in filesystem cephfs as rank 2 2023-07-14T11:10:15.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.355+0000 7f1819ffb700 1 -- 172.21.15.40:0/2049151387 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]=0 v22) v1 ==== 115+0+0 (secure 0 0 0) 0x7f1814019070 con 0x7f181c101660 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 msgr2=0x7f18080a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 0x7f18080a3680 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f180c006010 tx=0x7f180c005960 comp rx=0 tx=0).stop 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 msgr2=0x7f181c072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 0x7f181c072e40 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f1814007f10 tx=0x7f181400d3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 shutdown_connections 2023-07-14T11:10:15.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18080a11d0 0x7f18080a3680 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f181c101660 0x7f181c072e40 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f181c0ffae0 0x7f181c0742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 --2- 172.21.15.40:0/2049151387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f181c100cd0 0x7f181c072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 >> 172.21.15.40:0/2049151387 conn(0x7f181c0fb0e0 msgr2=0x7f181c104260 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 shutdown_connections 2023-07-14T11:10:15.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.358+0000 7f18225ff700 1 -- 172.21.15.40:0/2049151387 wait complete. 2023-07-14T11:10:15.383 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2023-07-14T11:10:15.383 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:10:15.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.586+0000 7fb5ecbff700 1 -- 172.21.15.40:0/1149537046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e80ffb00 msgr2=0x7fb5e80fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.586+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/1149537046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e80ffb00 0x7fb5e80fff10 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7fb5d8009990 tx=0x7fb5d8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 -- 172.21.15.40:0/1149537046 shutdown_connections 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/1149537046 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e8101680 0x7fb5e8107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/1149537046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e80ffb00 0x7fb5e80fff10 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/1149537046 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 -- 172.21.15.40:0/1149537046 >> 172.21.15.40:0/1149537046 conn(0x7fb5e80fb0e0 msgr2=0x7fb5e80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 -- 172.21.15.40:0/1149537046 shutdown_connections 2023-07-14T11:10:15.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.587+0000 7fb5ecbff700 1 -- 172.21.15.40:0/1149537046 wait complete. 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 Processor -- start 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 -- start start 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8197f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 0x7fb5e819f000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:15.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5e819f5d0 con 0x7fb5e8101680 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5e819f710 con 0x7fb5e8100cf0 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.588+0000 7fb5ecbff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5e819f8a0 con 0x7fb5e80ffb00 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8197f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59460/0 (socket says 172.21.15.40:59460) 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 0x7fb5e819f000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:15.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8197f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38526/0 (socket says 172.21.15.40:38526) 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 -- 172.21.15.40:0/3664247467 learned_addr learned my addr 172.21.15.40:0/3664247467 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 -- 172.21.15.40:0/3664247467 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 msgr2=0x7fb5e8197f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8197f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 -- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 msgr2=0x7fb5e819f000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 0x7fb5e819f000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 -- 172.21.15.40:0/3664247467 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb5d80095f0 con 0x7fb5e80ffb00 2023-07-14T11:10:15.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e6d9d700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 0x7fb5e819f000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:15.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.589+0000 7fb5e659c700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fb5d8004730 tx=0x7fb5d8004760 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:15.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.590+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5d801d070 con 0x7fb5e80ffb00 2023-07-14T11:10:15.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.590+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb5d8021410 con 0x7fb5e80ffb00 2023-07-14T11:10:15.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.590+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb5e819fb20 con 0x7fb5e80ffb00 2023-07-14T11:10:15.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.590+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb5d8017650 con 0x7fb5e80ffb00 2023-07-14T11:10:15.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.590+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb5e819ffa0 con 0x7fb5e80ffb00 2023-07-14T11:10:15.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.591+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5e810b320 con 0x7fb5e80ffb00 2023-07-14T11:10:15.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.594+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb5d8021580 con 0x7fb5e80ffb00 2023-07-14T11:10:15.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.595+0000 7fb5d77fe700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 0x7fb5d00a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:15.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.596+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fb5d80ce670 con 0x7fb5e80ffb00 2023-07-14T11:10:15.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.596+0000 7fb5e5d9b700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 0x7fb5d00a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:15.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.597+0000 7fb5e5d9b700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 0x7fb5d00a36d0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc009c60 tx=0x7fb5dc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:15.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.598+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb5d8097ae0 con 0x7fb5e80ffb00 2023-07-14T11:10:15.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.853+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fb5e81a0550 con 0x7fb5e80ffb00 2023-07-14T11:10:15.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.854+0000 7fb5d77fe700 1 -- 172.21.15.40:0/3664247467 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 23 v23) v1 ==== 94+0+10849 (secure 0 0 0) 0x7fb5d809b100 con 0x7fb5e80ffb00 2023-07-14T11:10:15.856 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:15.856 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":23,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3},{"gid":24466,"name":"l","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24482,"name":"d","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":5},{"gid":24494,"name":"e","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":6},{"gid":24511,"name":"f","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":7}],"filesystems":[{"mdsmap":{"epoch":23,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:15.355684+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:creating","state_seq":1,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:10:15.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 msgr2=0x7fb5d00a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 0x7fb5d00a36d0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7fb5dc009c60 tx=0x7fb5dc009260 comp rx=0 tx=0).stop 2023-07-14T11:10:15.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 msgr2=0x7fb5e8197a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:15.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fb5d8004730 tx=0x7fb5d8004760 comp rx=0 tx=0).stop 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 shutdown_connections 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb5d00a1220 0x7fb5d00a36d0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5e8101680 0x7fb5e819f000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb5e80ffb00 0x7fb5e8197a50 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 --2- 172.21.15.40:0/3664247467 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb5e8100cf0 0x7fb5e8197f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.856+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 >> 172.21.15.40:0/3664247467 conn(0x7fb5e80fb0e0 msgr2=0x7fb5e80fd310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.857+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 shutdown_connections 2023-07-14T11:10:15.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:15.857+0000 7fb5ecbff700 1 -- 172.21.15.40:0/3664247467 wait complete. 2023-07-14T11:10:15.860 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 23 2023-07-14T11:10:15.876 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:10:16.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: pgmap v294: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 895 B/s rd, 255 B/s wr, 1 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:active 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: daemon mds.h assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:active} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:creating} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:15 smithi130 ceph-mon[129405]: daemon mds.h is now active in filesystem cephfs as rank 3 2023-07-14T11:10:16.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: pgmap v294: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 895 B/s rd, 255 B/s wr, 1 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:16.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:active 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: daemon mds.h assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:active} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:creating} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:15 smithi040 ceph-mon[97954]: daemon mds.h is now active in filesystem cephfs as rank 3 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: pgmap v294: 129 pgs: 129 active+clean; 581 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 895 B/s rd, 255 B/s wr, 1 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]: dispatch 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] up:active 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2049151387' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "session_timeout", "val": "300"}]': finished 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: daemon mds.h assigned to filesystem cephfs as rank 3 (now has 4 ranks) 2023-07-14T11:10:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: fsmap cephfs:3 {0=b=up:active,1=a=up:active,2=c=up:active} 2 up:standby-replay 7 up:standby 2023-07-14T11:10:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:creating} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:15 smithi045 ceph-mon[128791]: daemon mds.h is now active in filesystem cephfs as rank 3 2023-07-14T11:10:16.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 -- 172.21.15.40:0/3179518457 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c101680 msgr2=0x7f2c2c107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 --2- 172.21.15.40:0/3179518457 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c101680 0x7f2c2c107f40 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f2c2400b0a0 tx=0x7f2c2400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 -- 172.21.15.40:0/3179518457 shutdown_connections 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 --2- 172.21.15.40:0/3179518457 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c101680 0x7f2c2c107f40 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 --2- 172.21.15.40:0/3179518457 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 --2- 172.21.15.40:0/3179518457 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.079+0000 7f2c31dff700 1 -- 172.21.15.40:0/3179518457 >> 172.21.15.40:0/3179518457 conn(0x7f2c2c0fb0e0 msgr2=0x7f2c2c0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.080+0000 7f2c31dff700 1 -- 172.21.15.40:0/3179518457 shutdown_connections 2023-07-14T11:10:16.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.080+0000 7f2c31dff700 1 -- 172.21.15.40:0/3179518457 wait complete. 2023-07-14T11:10:16.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 Processor -- start 2023-07-14T11:10:16.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 -- start start 2023-07-14T11:10:16.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c101680 0x7f2c2c19f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c2c19f6c0 con 0x7f2c2c100cf0 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c2c19f800 con 0x7f2c2c101680 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c31dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c2c19f990 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c2b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c2affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c2b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59486/0 (socket says 172.21.15.40:59486) 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.081+0000 7f2c2b7fe700 1 -- 172.21.15.40:0/2389327160 learned_addr learned my addr 172.21.15.40:0/2389327160 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2bfff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c101680 0x7f2c2c19f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 -- 172.21.15.40:0/2389327160 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c101680 msgr2=0x7f2c2c19f0f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c101680 0x7f2c2c19f0f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 -- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 msgr2=0x7f2c2c198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 -- 172.21.15.40:0/2389327160 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2c24009d00 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2affd700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c2b7fe700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f2c1c00e970 tx=0x7f2c1c00ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c1c009790 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2c2c19fc70 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2c1c00c7e0 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.082+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c1c003c70 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.083+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2c2c1a01d0 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.083+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2c2c10b330 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.086+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f2c1c00cc50 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.087+0000 7f2c28ff9700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 0x7f2c140a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.087+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f2c1c014070 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.087+0000 7f2c2affd700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 0x7f2c140a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.088+0000 7f2c2affd700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 0x7f2c140a3650 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f2c20011490 tx=0x7f2c20009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:16.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.089+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f2c1c096370 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.298+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2c2c10b520 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.300+0000 7f2c28ff9700 1 -- 172.21.15.40:0/2389327160 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+24369 (secure 0 0 0) 0x7f2c1c099990 con 0x7f2c2c0ffb00 2023-07-14T11:10:16.302 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:16.303 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":90,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:10:07.729799+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:16.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 msgr2=0x7f2c140a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 0x7f2c140a3650 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f2c20011490 tx=0x7f2c20009260 comp rx=0 tx=0).stop 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 msgr2=0x7f2c2c197b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f2c1c00e970 tx=0x7f2c1c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 shutdown_connections 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c140a11a0 0x7f2c140a3650 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c2c101680 0x7f2c2c19f0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c2c0ffb00 0x7f2c2c197b40 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 --2- 172.21.15.40:0/2389327160 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c2c100cf0 0x7f2c2c198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 >> 172.21.15.40:0/2389327160 conn(0x7f2c2c0fb0e0 msgr2=0x7f2c2c1064d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:16.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 shutdown_connections 2023-07-14T11:10:16.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.303+0000 7f2c31dff700 1 -- 172.21.15.40:0/2389327160 wait complete. 2023-07-14T11:10:16.323 INFO:tasks.cephfs.filesystem.ceph_manager:waiting for clean 2023-07-14T11:10:16.323 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-07-14T11:10:16.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.530+0000 7efec5dff700 1 -- 172.21.15.40:0/154160153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec0100cf0 msgr2=0x7efec0101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.530+0000 7efec5dff700 1 --2- 172.21.15.40:0/154160153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec0100cf0 0x7efec0101140 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7efeb4009990 tx=0x7efeb4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 -- 172.21.15.40:0/154160153 shutdown_connections 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 --2- 172.21.15.40:0/154160153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0101680 0x7efec0107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 --2- 172.21.15.40:0/154160153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec00ffb00 0x7efec00fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 --2- 172.21.15.40:0/154160153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec0100cf0 0x7efec0101140 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 -- 172.21.15.40:0/154160153 >> 172.21.15.40:0/154160153 conn(0x7efec00fb0e0 msgr2=0x7efec00fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:16.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 -- 172.21.15.40:0/154160153 shutdown_connections 2023-07-14T11:10:16.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.531+0000 7efec5dff700 1 -- 172.21.15.40:0/154160153 wait complete. 2023-07-14T11:10:16.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 Processor -- start 2023-07-14T11:10:16.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 -- start start 2023-07-14T11:10:16.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0100cf0 0x7efec0197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec0101680 0x7efec019efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec019f590 con 0x7efec00ffb00 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec019f6d0 con 0x7efec0101680 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.532+0000 7efec5dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efec019f860 con 0x7efec0100cf0 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:43122/0 (socket says 172.21.15.40:43122) 2023-07-14T11:10:16.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 -- 172.21.15.40:0/2547789307 learned_addr learned my addr 172.21.15.40:0/2547789307 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebeffd700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0100cf0 0x7efec0197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebffff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec0101680 0x7efec019efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 -- 172.21.15.40:0/2547789307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0100cf0 msgr2=0x7efec0197f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0100cf0 0x7efec0197f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 -- 172.21.15.40:0/2547789307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec0101680 msgr2=0x7efec019efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec0101680 0x7efec019efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 -- 172.21.15.40:0/2547789307 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efeb0009520 con 0x7efec00ffb00 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.533+0000 7efebf7fe700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7efeb000ea50 tx=0x7efeb000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.534+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb000cbd0 con 0x7efec00ffb00 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.534+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efeb000cd30 con 0x7efec00ffb00 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.534+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efeb40095f0 con 0x7efec00ffb00 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.534+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efeb0018680 con 0x7efec00ffb00 2023-07-14T11:10:16.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.534+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efec019fec0 con 0x7efec00ffb00 2023-07-14T11:10:16.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.535+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efec010b350 con 0x7efec00ffb00 2023-07-14T11:10:16.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.537+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7efeb0010430 con 0x7efec00ffb00 2023-07-14T11:10:16.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.538+0000 7efebcff9700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 0x7efea80a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:16.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.538+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7efeb0014070 con 0x7efec00ffb00 2023-07-14T11:10:16.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.538+0000 7efebeffd700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 0x7efea80a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:16.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.539+0000 7efebeffd700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 0x7efea80a3650 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7efeb400b2e0 tx=0x7efeb4005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:16.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.542+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7efeb0096500 con 0x7efec00ffb00 2023-07-14T11:10:16.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.747+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7efec01075a0 con 0x7efea80a11a0 2023-07-14T11:10:16.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.762+0000 7efebcff9700 1 -- 172.21.15.40:0/2547789307 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+422290 (secure 0 0 0) 0x7efec01075a0 con 0x7efea80a11a0 2023-07-14T11:10:16.769 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 msgr2=0x7efea80a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 0x7efea80a3650 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7efeb400b2e0 tx=0x7efeb4005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 msgr2=0x7efec0197a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7efeb000ea50 tx=0x7efeb000ed60 comp rx=0 tx=0).stop 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 shutdown_connections 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7efea80a11a0 0x7efea80a3650 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efec0101680 0x7efec019efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efec00ffb00 0x7efec0197a10 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 --2- 172.21.15.40:0/2547789307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efec0100cf0 0x7efec0197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 >> 172.21.15.40:0/2547789307 conn(0x7efec00fb0e0 msgr2=0x7efec00fd310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 shutdown_connections 2023-07-14T11:10:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:16.768+0000 7efec5dff700 1 -- 172.21.15.40:0/2547789307 wait complete. 2023-07-14T11:10:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:10:16.789 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":295,"stamp":"2023-07-14T11:10:16.632543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596486,"num_objects":57,"num_object_clones":0,"num_object_copies":171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57,"num_whiteouts":0,"num_read":147,"num_read_kb":132,"num_write":301,"num_write_kb":2979,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":37,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":239,"ondisk_log_size":239,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337012,"kb_used_data":13588,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604812,"statfs":{"total":1151940427776,"available":1151595327488,"internally_reserved":0,"allocated":13914112,"data_stored":7651900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1667,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":11,"num_read_kb":13,"num_write":72,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.902015"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738359+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:10:07.738359+0000","last_peered":"2023-07-14T11:10:07.738359+0000","last_clean":"2023-07-14T11:10:07.738359+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:10:07.738359+0000","last_undegraded":"2023-07-14T11:10:07.738359+0000","last_fullsized":"2023-07-14T11:10:07.738359+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:21:10.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141620+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:10:08.141620+0000","last_peered":"2023-07-14T11:10:08.141620+0000","last_clean":"2023-07-14T11:10:08.141620+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:10:08.141620+0000","last_undegraded":"2023-07-14T11:10:08.141620+0000","last_fullsized":"2023-07-14T11:10:08.141620+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:43:13.354815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142202+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:10:08.142202+0000","last_peered":"2023-07-14T11:10:08.142202+0000","last_clean":"2023-07-14T11:10:08.142202+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:10:08.142202+0000","last_undegraded":"2023-07-14T11:10:08.142202+0000","last_fullsized":"2023-07-14T11:10:08.142202+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:25:21.721235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823979+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:10:07.823979+0000","last_peered":"2023-07-14T11:10:07.823979+0000","last_clean":"2023-07-14T11:10:07.823979+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:10:07.823979+0000","last_undegraded":"2023-07-14T11:10:07.823979+0000","last_fullsized":"2023-07-14T11:10:07.823979+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:50:39.156238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734553+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:10:07.734553+0000","last_peered":"2023-07-14T11:10:07.734553+0000","last_clean":"2023-07-14T11:10:07.734553+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:10:07.734553+0000","last_undegraded":"2023-07-14T11:10:07.734553+0000","last_fullsized":"2023-07-14T11:10:07.734553+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:08:20.010349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142872+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:10:08.142872+0000","last_peered":"2023-07-14T11:10:08.142872+0000","last_clean":"2023-07-14T11:10:08.142872+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:10:08.142872+0000","last_undegraded":"2023-07-14T11:10:08.142872+0000","last_fullsized":"2023-07-14T11:10:08.142872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:38:58.846023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736546+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:10:07.736546+0000","last_peered":"2023-07-14T11:10:07.736546+0000","last_clean":"2023-07-14T11:10:07.736546+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:10:07.736546+0000","last_undegraded":"2023-07-14T11:10:07.736546+0000","last_fullsized":"2023-07-14T11:10:07.736546+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:29:36.124497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739054+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:10:07.739054+0000","last_peered":"2023-07-14T11:10:07.739054+0000","last_clean":"2023-07-14T11:10:07.739054+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:10:07.739054+0000","last_undegraded":"2023-07-14T11:10:07.739054+0000","last_fullsized":"2023-07-14T11:10:07.739054+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:48:52.238878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735166+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:10:07.735166+0000","last_peered":"2023-07-14T11:10:07.735166+0000","last_clean":"2023-07-14T11:10:07.735166+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:10:07.735166+0000","last_undegraded":"2023-07-14T11:10:07.735166+0000","last_fullsized":"2023-07-14T11:10:07.735166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:45.770708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143166+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:10:08.143166+0000","last_peered":"2023-07-14T11:10:08.143166+0000","last_clean":"2023-07-14T11:10:08.143166+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:10:08.143166+0000","last_undegraded":"2023-07-14T11:10:08.143166+0000","last_fullsized":"2023-07-14T11:10:08.143166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:11.980357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737468+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:10:07.737468+0000","last_peered":"2023-07-14T11:10:07.737468+0000","last_clean":"2023-07-14T11:10:07.737468+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:10:07.737468+0000","last_undegraded":"2023-07-14T11:10:07.737468+0000","last_fullsized":"2023-07-14T11:10:07.737468+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:31:39.003266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823717+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:10:07.823717+0000","last_peered":"2023-07-14T11:10:07.823717+0000","last_clean":"2023-07-14T11:10:07.823717+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:10:07.823717+0000","last_undegraded":"2023-07-14T11:10:07.823717+0000","last_fullsized":"2023-07-14T11:10:07.823717+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:03:29.844122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734676+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:10:07.734676+0000","last_peered":"2023-07-14T11:10:07.734676+0000","last_clean":"2023-07-14T11:10:07.734676+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:10:07.734676+0000","last_undegraded":"2023-07-14T11:10:07.734676+0000","last_fullsized":"2023-07-14T11:10:07.734676+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:10:43.308729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141270+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:10:08.141270+0000","last_peered":"2023-07-14T11:10:08.141270+0000","last_clean":"2023-07-14T11:10:08.141270+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:10:08.141270+0000","last_undegraded":"2023-07-14T11:10:08.141270+0000","last_fullsized":"2023-07-14T11:10:08.141270+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:20:00.054130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141301+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:10:08.141301+0000","last_peered":"2023-07-14T11:10:08.141301+0000","last_clean":"2023-07-14T11:10:08.141301+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:10:08.141301+0000","last_undegraded":"2023-07-14T11:10:08.141301+0000","last_fullsized":"2023-07-14T11:10:08.141301+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:38:22.587911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734282+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:10:07.734282+0000","last_peered":"2023-07-14T11:10:07.734282+0000","last_clean":"2023-07-14T11:10:07.734282+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:10:07.734282+0000","last_undegraded":"2023-07-14T11:10:07.734282+0000","last_fullsized":"2023-07-14T11:10:07.734282+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:53:57.650717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738135+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:10:07.738135+0000","last_peered":"2023-07-14T11:10:07.738135+0000","last_clean":"2023-07-14T11:10:07.738135+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:10:07.738135+0000","last_undegraded":"2023-07-14T11:10:07.738135+0000","last_fullsized":"2023-07-14T11:10:07.738135+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:48:17.654653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740775+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:10:07.740775+0000","last_peered":"2023-07-14T11:10:07.740775+0000","last_clean":"2023-07-14T11:10:07.740775+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:10:07.740775+0000","last_undegraded":"2023-07-14T11:10:07.740775+0000","last_fullsized":"2023-07-14T11:10:07.740775+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:36:51.637622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737923+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:10:07.737923+0000","last_peered":"2023-07-14T11:10:07.737923+0000","last_clean":"2023-07-14T11:10:07.737923+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:07.737923+0000","last_undegraded":"2023-07-14T11:10:07.737923+0000","last_fullsized":"2023-07-14T11:10:07.737923+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:45.243063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738752+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:10:07.738752+0000","last_peered":"2023-07-14T11:10:07.738752+0000","last_clean":"2023-07-14T11:10:07.738752+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:10:07.738752+0000","last_undegraded":"2023-07-14T11:10:07.738752+0000","last_fullsized":"2023-07-14T11:10:07.738752+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:12:05.905433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"83'4","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740661+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:10:07.740661+0000","last_peered":"2023-07-14T11:10:07.740661+0000","last_clean":"2023-07-14T11:10:07.740661+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:10:07.740661+0000","last_undegraded":"2023-07-14T11:10:07.740661+0000","last_fullsized":"2023-07-14T11:10:07.740661+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:21:25.983167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:53.251986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822942+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:10:07.822942+0000","last_peered":"2023-07-14T11:10:07.822942+0000","last_clean":"2023-07-14T11:10:07.822942+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:10:07.822942+0000","last_undegraded":"2023-07-14T11:10:07.822942+0000","last_fullsized":"2023-07-14T11:10:07.822942+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:32:00.530574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740281+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:10:07.740281+0000","last_peered":"2023-07-14T11:10:07.740281+0000","last_clean":"2023-07-14T11:10:07.740281+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:10:07.740281+0000","last_undegraded":"2023-07-14T11:10:07.740281+0000","last_fullsized":"2023-07-14T11:10:07.740281+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:56.375572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142002+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:10:08.142002+0000","last_peered":"2023-07-14T11:10:08.142002+0000","last_clean":"2023-07-14T11:10:08.142002+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:10:08.142002+0000","last_undegraded":"2023-07-14T11:10:08.142002+0000","last_fullsized":"2023-07-14T11:10:08.142002+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:26.963699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823419+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:10:07.823419+0000","last_peered":"2023-07-14T11:10:07.823419+0000","last_clean":"2023-07-14T11:10:07.823419+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:10:07.823419+0000","last_undegraded":"2023-07-14T11:10:07.823419+0000","last_fullsized":"2023-07-14T11:10:07.823419+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:43.101537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'11","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375838+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:10:16.375838+0000","last_peered":"2023-07-14T11:10:16.375838+0000","last_clean":"2023-07-14T11:10:16.375838+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:10:16.375838+0000","last_undegraded":"2023-07-14T11:10:16.375838+0000","last_fullsized":"2023-07-14T11:10:16.375838+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:26:07.506765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734736+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:10:07.734736+0000","last_peered":"2023-07-14T11:10:07.734736+0000","last_clean":"2023-07-14T11:10:07.734736+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:10:07.734736+0000","last_undegraded":"2023-07-14T11:10:07.734736+0000","last_fullsized":"2023-07-14T11:10:07.734736+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:10:30.512188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822955+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:10:07.822955+0000","last_peered":"2023-07-14T11:10:07.822955+0000","last_clean":"2023-07-14T11:10:07.822955+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:10:07.822955+0000","last_undegraded":"2023-07-14T11:10:07.822955+0000","last_fullsized":"2023-07-14T11:10:07.822955+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:44.974006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376472+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:10:16.376472+0000","last_peered":"2023-07-14T11:10:16.376472+0000","last_clean":"2023-07-14T11:10:16.376472+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:10:16.376472+0000","last_undegraded":"2023-07-14T11:10:16.376472+0000","last_fullsized":"2023-07-14T11:10:16.376472+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:41:26.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740246+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:10:07.740246+0000","last_peered":"2023-07-14T11:10:07.740246+0000","last_clean":"2023-07-14T11:10:07.740246+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:10:07.740246+0000","last_undegraded":"2023-07-14T11:10:07.740246+0000","last_fullsized":"2023-07-14T11:10:07.740246+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:32.963491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738142+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:10:07.738142+0000","last_peered":"2023-07-14T11:10:07.738142+0000","last_clean":"2023-07-14T11:10:07.738142+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:10:07.738142+0000","last_undegraded":"2023-07-14T11:10:07.738142+0000","last_fullsized":"2023-07-14T11:10:07.738142+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:51:58.731259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739304+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:10:07.739304+0000","last_peered":"2023-07-14T11:10:07.739304+0000","last_clean":"2023-07-14T11:10:07.739304+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:10:07.739304+0000","last_undegraded":"2023-07-14T11:10:07.739304+0000","last_fullsized":"2023-07-14T11:10:07.739304+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:06:48.309845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734873+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:10:07.734873+0000","last_peered":"2023-07-14T11:10:07.734873+0000","last_clean":"2023-07-14T11:10:07.734873+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:10:07.734873+0000","last_undegraded":"2023-07-14T11:10:07.734873+0000","last_fullsized":"2023-07-14T11:10:07.734873+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:48.090320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738348+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:10:07.738348+0000","last_peered":"2023-07-14T11:10:07.738348+0000","last_clean":"2023-07-14T11:10:07.738348+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:10:07.738348+0000","last_undegraded":"2023-07-14T11:10:07.738348+0000","last_fullsized":"2023-07-14T11:10:07.738348+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:36:58.104207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357621+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:10:14.357621+0000","last_peered":"2023-07-14T11:10:14.357621+0000","last_clean":"2023-07-14T11:10:14.357621+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:10:14.357621+0000","last_undegraded":"2023-07-14T11:10:14.357621+0000","last_fullsized":"2023-07-14T11:10:14.357621+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:38:46.335485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.745285+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:10:07.745285+0000","last_peered":"2023-07-14T11:10:07.745285+0000","last_clean":"2023-07-14T11:10:07.745285+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:10:07.745285+0000","last_undegraded":"2023-07-14T11:10:07.745285+0000","last_fullsized":"2023-07-14T11:10:07.745285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:17.289951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737973+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:10:07.737973+0000","last_peered":"2023-07-14T11:10:07.737973+0000","last_clean":"2023-07-14T11:10:07.737973+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:10:07.737973+0000","last_undegraded":"2023-07-14T11:10:07.737973+0000","last_fullsized":"2023-07-14T11:10:07.737973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:07:16.888150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734417+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:10:07.734417+0000","last_peered":"2023-07-14T11:10:07.734417+0000","last_clean":"2023-07-14T11:10:07.734417+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:10:07.734417+0000","last_undegraded":"2023-07-14T11:10:07.734417+0000","last_fullsized":"2023-07-14T11:10:07.734417+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:53:35.630621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738872+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:10:07.738872+0000","last_peered":"2023-07-14T11:10:07.738872+0000","last_clean":"2023-07-14T11:10:07.738872+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:10:07.738872+0000","last_undegraded":"2023-07-14T11:10:07.738872+0000","last_fullsized":"2023-07-14T11:10:07.738872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:01:08.293055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141456+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:10:08.141456+0000","last_peered":"2023-07-14T11:10:08.141456+0000","last_clean":"2023-07-14T11:10:08.141456+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:10:08.141456+0000","last_undegraded":"2023-07-14T11:10:08.141456+0000","last_fullsized":"2023-07-14T11:10:08.141456+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:14:59.191235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739475+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:10:07.739475+0000","last_peered":"2023-07-14T11:10:07.739475+0000","last_clean":"2023-07-14T11:10:07.739475+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:10:07.739475+0000","last_undegraded":"2023-07-14T11:10:07.739475+0000","last_fullsized":"2023-07-14T11:10:07.739475+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:39:15.619767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738091+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:10:07.738091+0000","last_peered":"2023-07-14T11:10:07.738091+0000","last_clean":"2023-07-14T11:10:07.738091+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:10:07.738091+0000","last_undegraded":"2023-07-14T11:10:07.738091+0000","last_fullsized":"2023-07-14T11:10:07.738091+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:48:41.840080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739132+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:10:07.739132+0000","last_peered":"2023-07-14T11:10:07.739132+0000","last_clean":"2023-07-14T11:10:07.739132+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:10:07.739132+0000","last_undegraded":"2023-07-14T11:10:07.739132+0000","last_fullsized":"2023-07-14T11:10:07.739132+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:39:26.932728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783768+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:10:13.783768+0000","last_peered":"2023-07-14T11:10:13.783768+0000","last_clean":"2023-07-14T11:10:13.783768+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:10:13.783768+0000","last_undegraded":"2023-07-14T11:10:13.783768+0000","last_fullsized":"2023-07-14T11:10:13.783768+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:10:05.049869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735216+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:10:07.735216+0000","last_peered":"2023-07-14T11:10:07.735216+0000","last_clean":"2023-07-14T11:10:07.735216+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:10:07.735216+0000","last_undegraded":"2023-07-14T11:10:07.735216+0000","last_fullsized":"2023-07-14T11:10:07.735216+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:50.720818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737634+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:10:07.737634+0000","last_peered":"2023-07-14T11:10:07.737634+0000","last_clean":"2023-07-14T11:10:07.737634+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:10:07.737634+0000","last_undegraded":"2023-07-14T11:10:07.737634+0000","last_fullsized":"2023-07-14T11:10:07.737634+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:25:31.401998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783874+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:10:13.783874+0000","last_peered":"2023-07-14T11:10:13.783874+0000","last_clean":"2023-07-14T11:10:13.783874+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:10:13.783874+0000","last_undegraded":"2023-07-14T11:10:13.783874+0000","last_fullsized":"2023-07-14T11:10:13.783874+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:10:40.132095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142089+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:10:08.142089+0000","last_peered":"2023-07-14T11:10:08.142089+0000","last_clean":"2023-07-14T11:10:08.142089+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:10:08.142089+0000","last_undegraded":"2023-07-14T11:10:08.142089+0000","last_fullsized":"2023-07-14T11:10:08.142089+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:56:07.808822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823945+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:10:07.823945+0000","last_peered":"2023-07-14T11:10:07.823945+0000","last_clean":"2023-07-14T11:10:07.823945+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:10:07.823945+0000","last_undegraded":"2023-07-14T11:10:07.823945+0000","last_fullsized":"2023-07-14T11:10:07.823945+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:16:13.828703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"83'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740105+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:10:07.740105+0000","last_peered":"2023-07-14T11:10:07.740105+0000","last_clean":"2023-07-14T11:10:07.740105+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:10:07.740105+0000","last_undegraded":"2023-07-14T11:10:07.740105+0000","last_fullsized":"2023-07-14T11:10:07.740105+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:30:17.168629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738962+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:10:07.738962+0000","last_peered":"2023-07-14T11:10:07.738962+0000","last_clean":"2023-07-14T11:10:07.738962+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:10:07.738962+0000","last_undegraded":"2023-07-14T11:10:07.738962+0000","last_fullsized":"2023-07-14T11:10:07.738962+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:25:22.187958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737948+0000","last_change":"2023-07-14T11:09:47.624060+0000","last_active":"2023-07-14T11:10:07.737948+0000","last_peered":"2023-07-14T11:10:07.737948+0000","last_clean":"2023-07-14T11:10:07.737948+0000","last_became_active":"2023-07-14T11:09:47.623605+0000","last_became_peered":"2023-07-14T11:09:47.623605+0000","last_unstale":"2023-07-14T11:10:07.737948+0000","last_undegraded":"2023-07-14T11:10:07.737948+0000","last_fullsized":"2023-07-14T11:10:07.737948+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:17:15.881693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"90'12","reported_seq":42,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.352417+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:10:14.352417+0000","last_peered":"2023-07-14T11:10:14.352417+0000","last_clean":"2023-07-14T11:10:14.352417+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:10:14.352417+0000","last_undegraded":"2023-07-14T11:10:14.352417+0000","last_fullsized":"2023-07-14T11:10:14.352417+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:01:52.717646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3794,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":6,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734684+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:10:07.734684+0000","last_peered":"2023-07-14T11:10:07.734684+0000","last_clean":"2023-07-14T11:10:07.734684+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:10:07.734684+0000","last_undegraded":"2023-07-14T11:10:07.734684+0000","last_fullsized":"2023-07-14T11:10:07.734684+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:55.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740741+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:10:07.740741+0000","last_peered":"2023-07-14T11:10:07.740741+0000","last_clean":"2023-07-14T11:10:07.740741+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:10:07.740741+0000","last_undegraded":"2023-07-14T11:10:07.740741+0000","last_fullsized":"2023-07-14T11:10:07.740741+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:37.113063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"83'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737743+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:10:07.737743+0000","last_peered":"2023-07-14T11:10:07.737743+0000","last_clean":"2023-07-14T11:10:07.737743+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:10:07.737743+0000","last_undegraded":"2023-07-14T11:10:07.737743+0000","last_fullsized":"2023-07-14T11:10:07.737743+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:39:38.915458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734456+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:10:07.734456+0000","last_peered":"2023-07-14T11:10:07.734456+0000","last_clean":"2023-07-14T11:10:07.734456+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:10:07.734456+0000","last_undegraded":"2023-07-14T11:10:07.734456+0000","last_fullsized":"2023-07-14T11:10:07.734456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:25:56.604869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744522+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:10:07.744522+0000","last_peered":"2023-07-14T11:10:07.744522+0000","last_clean":"2023-07-14T11:10:07.744522+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:10:07.744522+0000","last_undegraded":"2023-07-14T11:10:07.744522+0000","last_fullsized":"2023-07-14T11:10:07.744522+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:08.719446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741523+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:10:07.741523+0000","last_peered":"2023-07-14T11:10:07.741523+0000","last_clean":"2023-07-14T11:10:07.741523+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:10:07.741523+0000","last_undegraded":"2023-07-14T11:10:07.741523+0000","last_fullsized":"2023-07-14T11:10:07.741523+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:12:45.811503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736758+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:10:07.736758+0000","last_peered":"2023-07-14T11:10:07.736758+0000","last_clean":"2023-07-14T11:10:07.736758+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:10:07.736758+0000","last_undegraded":"2023-07-14T11:10:07.736758+0000","last_fullsized":"2023-07-14T11:10:07.736758+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:28.882720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740995+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:10:07.740995+0000","last_peered":"2023-07-14T11:10:07.740995+0000","last_clean":"2023-07-14T11:10:07.740995+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:10:07.740995+0000","last_undegraded":"2023-07-14T11:10:07.740995+0000","last_fullsized":"2023-07-14T11:10:07.740995+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:35.770995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.368124+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:10:15.368124+0000","last_peered":"2023-07-14T11:10:15.368124+0000","last_clean":"2023-07-14T11:10:15.368124+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:10:15.368124+0000","last_undegraded":"2023-07-14T11:10:15.368124+0000","last_fullsized":"2023-07-14T11:10:15.368124+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:57:32.688631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737387+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:10:07.737387+0000","last_peered":"2023-07-14T11:10:07.737387+0000","last_clean":"2023-07-14T11:10:07.737387+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:10:07.737387+0000","last_undegraded":"2023-07-14T11:10:07.737387+0000","last_fullsized":"2023-07-14T11:10:07.737387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:18:06.626041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734847+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:10:07.734847+0000","last_peered":"2023-07-14T11:10:07.734847+0000","last_clean":"2023-07-14T11:10:07.734847+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:10:07.734847+0000","last_undegraded":"2023-07-14T11:10:07.734847+0000","last_fullsized":"2023-07-14T11:10:07.734847+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:14:05.500061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":43,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369490+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:10:15.369490+0000","last_peered":"2023-07-14T11:10:15.369490+0000","last_clean":"2023-07-14T11:10:15.369490+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:10:15.369490+0000","last_undegraded":"2023-07-14T11:10:15.369490+0000","last_fullsized":"2023-07-14T11:10:15.369490+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:26:26.993318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741882+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:10:07.741882+0000","last_peered":"2023-07-14T11:10:07.741882+0000","last_clean":"2023-07-14T11:10:07.741882+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:10:07.741882+0000","last_undegraded":"2023-07-14T11:10:07.741882+0000","last_fullsized":"2023-07-14T11:10:07.741882+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:38:12.515688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741815+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:10:07.741815+0000","last_peered":"2023-07-14T11:10:07.741815+0000","last_clean":"2023-07-14T11:10:07.741815+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:10:07.741815+0000","last_undegraded":"2023-07-14T11:10:07.741815+0000","last_fullsized":"2023-07-14T11:10:07.741815+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:39.325247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"83'1","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734769+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:10:07.734769+0000","last_peered":"2023-07-14T11:10:07.734769+0000","last_clean":"2023-07-14T11:10:07.734769+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:10:07.734769+0000","last_undegraded":"2023-07-14T11:10:07.734769+0000","last_fullsized":"2023-07-14T11:10:07.734769+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:16.215502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738122+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:10:07.738122+0000","last_peered":"2023-07-14T11:10:07.738122+0000","last_clean":"2023-07-14T11:10:07.738122+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:10:07.738122+0000","last_undegraded":"2023-07-14T11:10:07.738122+0000","last_fullsized":"2023-07-14T11:10:07.738122+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:28.226381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734949+0000","last_change":"2023-07-14T11:09:47.736950+0000","last_active":"2023-07-14T11:10:07.734949+0000","last_peered":"2023-07-14T11:10:07.734949+0000","last_clean":"2023-07-14T11:10:07.734949+0000","last_became_active":"2023-07-14T11:09:47.736506+0000","last_became_peered":"2023-07-14T11:09:47.736506+0000","last_unstale":"2023-07-14T11:10:07.734949+0000","last_undegraded":"2023-07-14T11:10:07.734949+0000","last_fullsized":"2023-07-14T11:10:07.734949+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:24.567987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738616+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:10:07.738616+0000","last_peered":"2023-07-14T11:10:07.738616+0000","last_clean":"2023-07-14T11:10:07.738616+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:10:07.738616+0000","last_undegraded":"2023-07-14T11:10:07.738616+0000","last_fullsized":"2023-07-14T11:10:07.738616+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:11:19.303557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142811+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:10:08.142811+0000","last_peered":"2023-07-14T11:10:08.142811+0000","last_clean":"2023-07-14T11:10:08.142811+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:08.142811+0000","last_undegraded":"2023-07-14T11:10:08.142811+0000","last_fullsized":"2023-07-14T11:10:08.142811+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:51:22.856363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735220+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:10:07.735220+0000","last_peered":"2023-07-14T11:10:07.735220+0000","last_clean":"2023-07-14T11:10:07.735220+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:10:07.735220+0000","last_undegraded":"2023-07-14T11:10:07.735220+0000","last_fullsized":"2023-07-14T11:10:07.735220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:52:35.457076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736779+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:10:07.736779+0000","last_peered":"2023-07-14T11:10:07.736779+0000","last_clean":"2023-07-14T11:10:07.736779+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:10:07.736779+0000","last_undegraded":"2023-07-14T11:10:07.736779+0000","last_fullsized":"2023-07-14T11:10:07.736779+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:49:44.032115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143050+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:10:08.143050+0000","last_peered":"2023-07-14T11:10:08.143050+0000","last_clean":"2023-07-14T11:10:08.143050+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:10:08.143050+0000","last_undegraded":"2023-07-14T11:10:08.143050+0000","last_fullsized":"2023-07-14T11:10:08.143050+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:04:11.876243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734819+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:10:07.734819+0000","last_peered":"2023-07-14T11:10:07.734819+0000","last_clean":"2023-07-14T11:10:07.734819+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:10:07.734819+0000","last_undegraded":"2023-07-14T11:10:07.734819+0000","last_fullsized":"2023-07-14T11:10:07.734819+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:26:53.174066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739718+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:10:07.739718+0000","last_peered":"2023-07-14T11:10:07.739718+0000","last_clean":"2023-07-14T11:10:07.739718+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:10:07.739718+0000","last_undegraded":"2023-07-14T11:10:07.739718+0000","last_fullsized":"2023-07-14T11:10:07.739718+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:16:20.330278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734207+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:10:07.734207+0000","last_peered":"2023-07-14T11:10:07.734207+0000","last_clean":"2023-07-14T11:10:07.734207+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:10:07.734207+0000","last_undegraded":"2023-07-14T11:10:07.734207+0000","last_fullsized":"2023-07-14T11:10:07.734207+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:50:46.452985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736543+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:10:07.736543+0000","last_peered":"2023-07-14T11:10:07.736543+0000","last_clean":"2023-07-14T11:10:07.736543+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:10:07.736543+0000","last_undegraded":"2023-07-14T11:10:07.736543+0000","last_fullsized":"2023-07-14T11:10:07.736543+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:56:32.791400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738925+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:10:07.738925+0000","last_peered":"2023-07-14T11:10:07.738925+0000","last_clean":"2023-07-14T11:10:07.738925+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:10:07.738925+0000","last_undegraded":"2023-07-14T11:10:07.738925+0000","last_fullsized":"2023-07-14T11:10:07.738925+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:02:28.884240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.367881+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:10:15.367881+0000","last_peered":"2023-07-14T11:10:15.367881+0000","last_clean":"2023-07-14T11:10:15.367881+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:10:15.367881+0000","last_undegraded":"2023-07-14T11:10:15.367881+0000","last_fullsized":"2023-07-14T11:10:15.367881+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:12:29.230412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":354,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736416+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:10:07.736416+0000","last_peered":"2023-07-14T11:10:07.736416+0000","last_clean":"2023-07-14T11:10:07.736416+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:10:07.736416+0000","last_undegraded":"2023-07-14T11:10:07.736416+0000","last_fullsized":"2023-07-14T11:10:07.736416+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:40:53.088449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741053+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:10:07.741053+0000","last_peered":"2023-07-14T11:10:07.741053+0000","last_clean":"2023-07-14T11:10:07.741053+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:10:07.741053+0000","last_undegraded":"2023-07-14T11:10:07.741053+0000","last_fullsized":"2023-07-14T11:10:07.741053+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:46:21.793891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734132+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:10:07.734132+0000","last_peered":"2023-07-14T11:10:07.734132+0000","last_clean":"2023-07-14T11:10:07.734132+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:10:07.734132+0000","last_undegraded":"2023-07-14T11:10:07.734132+0000","last_fullsized":"2023-07-14T11:10:07.734132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:58:44.611438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'3","reported_seq":35,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357053+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:10:14.357053+0000","last_peered":"2023-07-14T11:10:14.357053+0000","last_clean":"2023-07-14T11:10:14.357053+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:10:14.357053+0000","last_undegraded":"2023-07-14T11:10:14.357053+0000","last_fullsized":"2023-07-14T11:10:14.357053+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:06:23.046904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737806+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:10:07.737806+0000","last_peered":"2023-07-14T11:10:07.737806+0000","last_clean":"2023-07-14T11:10:07.737806+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:10:07.737806+0000","last_undegraded":"2023-07-14T11:10:07.737806+0000","last_fullsized":"2023-07-14T11:10:07.737806+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:15.765387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.370106+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:10:15.370106+0000","last_peered":"2023-07-14T11:10:15.370106+0000","last_clean":"2023-07-14T11:10:15.370106+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:10:15.370106+0000","last_undegraded":"2023-07-14T11:10:15.370106+0000","last_fullsized":"2023-07-14T11:10:15.370106+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:27:03.761811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734605+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:10:07.734605+0000","last_peered":"2023-07-14T11:10:07.734605+0000","last_clean":"2023-07-14T11:10:07.734605+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:10:07.734605+0000","last_undegraded":"2023-07-14T11:10:07.734605+0000","last_fullsized":"2023-07-14T11:10:07.734605+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:17:38.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:57.136733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"87'1","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.748353+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:10:07.748353+0000","last_peered":"2023-07-14T11:10:07.748353+0000","last_clean":"2023-07-14T11:10:07.748353+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:10:07.748353+0000","last_undegraded":"2023-07-14T11:10:07.748353+0000","last_fullsized":"2023-07-14T11:10:07.748353+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:47:01.844602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740024+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:10:07.740024+0000","last_peered":"2023-07-14T11:10:07.740024+0000","last_clean":"2023-07-14T11:10:07.740024+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:10:07.740024+0000","last_undegraded":"2023-07-14T11:10:07.740024+0000","last_fullsized":"2023-07-14T11:10:07.740024+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:34:33.255530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738545+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:10:07.738545+0000","last_peered":"2023-07-14T11:10:07.738545+0000","last_clean":"2023-07-14T11:10:07.738545+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:10:07.738545+0000","last_undegraded":"2023-07-14T11:10:07.738545+0000","last_fullsized":"2023-07-14T11:10:07.738545+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:36.240727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375131+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:10:16.375131+0000","last_peered":"2023-07-14T11:10:16.375131+0000","last_clean":"2023-07-14T11:10:16.375131+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:10:16.375131+0000","last_undegraded":"2023-07-14T11:10:16.375131+0000","last_fullsized":"2023-07-14T11:10:16.375131+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:02:18.139490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141864+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:10:08.141864+0000","last_peered":"2023-07-14T11:10:08.141864+0000","last_clean":"2023-07-14T11:10:08.141864+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:10:08.141864+0000","last_undegraded":"2023-07-14T11:10:08.141864+0000","last_fullsized":"2023-07-14T11:10:08.141864+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:28:33.937520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823662+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:10:07.823662+0000","last_peered":"2023-07-14T11:10:07.823662+0000","last_clean":"2023-07-14T11:10:07.823662+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:10:07.823662+0000","last_undegraded":"2023-07-14T11:10:07.823662+0000","last_fullsized":"2023-07-14T11:10:07.823662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:28:39.306429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.782018+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:10:13.782018+0000","last_peered":"2023-07-14T11:10:13.782018+0000","last_clean":"2023-07-14T11:10:13.782018+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:10:13.782018+0000","last_undegraded":"2023-07-14T11:10:13.782018+0000","last_fullsized":"2023-07-14T11:10:13.782018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:50:14.638370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734307+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:10:07.734307+0000","last_peered":"2023-07-14T11:10:07.734307+0000","last_clean":"2023-07-14T11:10:07.734307+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:10:07.734307+0000","last_undegraded":"2023-07-14T11:10:07.734307+0000","last_fullsized":"2023-07-14T11:10:07.734307+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:27:40.728690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740731+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:10:07.740731+0000","last_peered":"2023-07-14T11:10:07.740731+0000","last_clean":"2023-07-14T11:10:07.740731+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:10:07.740731+0000","last_undegraded":"2023-07-14T11:10:07.740731+0000","last_fullsized":"2023-07-14T11:10:07.740731+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:37.925348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739281+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:10:07.739281+0000","last_peered":"2023-07-14T11:10:07.739281+0000","last_clean":"2023-07-14T11:10:07.739281+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:10:07.739281+0000","last_undegraded":"2023-07-14T11:10:07.739281+0000","last_fullsized":"2023-07-14T11:10:07.739281+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:35:40.777893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737586+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:10:07.737586+0000","last_peered":"2023-07-14T11:10:07.737586+0000","last_clean":"2023-07-14T11:10:07.737586+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:10:07.737586+0000","last_undegraded":"2023-07-14T11:10:07.737586+0000","last_fullsized":"2023-07-14T11:10:07.737586+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:43.286237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823509+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:10:07.823509+0000","last_peered":"2023-07-14T11:10:07.823509+0000","last_clean":"2023-07-14T11:10:07.823509+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:10:07.823509+0000","last_undegraded":"2023-07-14T11:10:07.823509+0000","last_fullsized":"2023-07-14T11:10:07.823509+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:28:35.378022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738408+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:10:07.738408+0000","last_peered":"2023-07-14T11:10:07.738408+0000","last_clean":"2023-07-14T11:10:07.738408+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:10:07.738408+0000","last_undegraded":"2023-07-14T11:10:07.738408+0000","last_fullsized":"2023-07-14T11:10:07.738408+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:10.923637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739938+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:10:07.739938+0000","last_peered":"2023-07-14T11:10:07.739938+0000","last_clean":"2023-07-14T11:10:07.739938+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:10:07.739938+0000","last_undegraded":"2023-07-14T11:10:07.739938+0000","last_fullsized":"2023-07-14T11:10:07.739938+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:27:36.946955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823764+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:10:07.823764+0000","last_peered":"2023-07-14T11:10:07.823764+0000","last_clean":"2023-07-14T11:10:07.823764+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:10:07.823764+0000","last_undegraded":"2023-07-14T11:10:07.823764+0000","last_fullsized":"2023-07-14T11:10:07.823764+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:46:25.967420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783411+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:10:13.783411+0000","last_peered":"2023-07-14T11:10:13.783411+0000","last_clean":"2023-07-14T11:10:13.783411+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:10:13.783411+0000","last_undegraded":"2023-07-14T11:10:13.783411+0000","last_fullsized":"2023-07-14T11:10:13.783411+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:08:54.674537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736919+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:10:07.736919+0000","last_peered":"2023-07-14T11:10:07.736919+0000","last_clean":"2023-07-14T11:10:07.736919+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:10:07.736919+0000","last_undegraded":"2023-07-14T11:10:07.736919+0000","last_fullsized":"2023-07-14T11:10:07.736919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:34:43.143864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734396+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:10:07.734396+0000","last_peered":"2023-07-14T11:10:07.734396+0000","last_clean":"2023-07-14T11:10:07.734396+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:10:07.734396+0000","last_undegraded":"2023-07-14T11:10:07.734396+0000","last_fullsized":"2023-07-14T11:10:07.734396+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:13:42.740364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738655+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:10:07.738655+0000","last_peered":"2023-07-14T11:10:07.738655+0000","last_clean":"2023-07-14T11:10:07.738655+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:10:07.738655+0000","last_undegraded":"2023-07-14T11:10:07.738655+0000","last_fullsized":"2023-07-14T11:10:07.738655+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:17:09.808452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737674+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:10:07.737674+0000","last_peered":"2023-07-14T11:10:07.737674+0000","last_clean":"2023-07-14T11:10:07.737674+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:10:07.737674+0000","last_undegraded":"2023-07-14T11:10:07.737674+0000","last_fullsized":"2023-07-14T11:10:07.737674+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:51:08.137181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141293+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:10:08.141293+0000","last_peered":"2023-07-14T11:10:08.141293+0000","last_clean":"2023-07-14T11:10:08.141293+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:10:08.141293+0000","last_undegraded":"2023-07-14T11:10:08.141293+0000","last_fullsized":"2023-07-14T11:10:08.141293+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:33:53.668586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740367+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:10:07.740367+0000","last_peered":"2023-07-14T11:10:07.740367+0000","last_clean":"2023-07-14T11:10:07.740367+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:10:07.740367+0000","last_undegraded":"2023-07-14T11:10:07.740367+0000","last_fullsized":"2023-07-14T11:10:07.740367+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:41:36.321965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736750+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:10:07.736750+0000","last_peered":"2023-07-14T11:10:07.736750+0000","last_clean":"2023-07-14T11:10:07.736750+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:10:07.736750+0000","last_undegraded":"2023-07-14T11:10:07.736750+0000","last_fullsized":"2023-07-14T11:10:07.736750+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:11:06.323972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739174+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:10:07.739174+0000","last_peered":"2023-07-14T11:10:07.739174+0000","last_clean":"2023-07-14T11:10:07.739174+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:10:07.739174+0000","last_undegraded":"2023-07-14T11:10:07.739174+0000","last_fullsized":"2023-07-14T11:10:07.739174+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:16.916714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"83'2","reported_seq":38,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.827927+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:10:07.827927+0000","last_peered":"2023-07-14T11:10:07.827927+0000","last_clean":"2023-07-14T11:10:07.827927+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:10:07.827927+0000","last_undegraded":"2023-07-14T11:10:07.827927+0000","last_fullsized":"2023-07-14T11:10:07.827927+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:33.874908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737919+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:10:07.737919+0000","last_peered":"2023-07-14T11:10:07.737919+0000","last_clean":"2023-07-14T11:10:07.737919+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:10:07.737919+0000","last_undegraded":"2023-07-14T11:10:07.737919+0000","last_fullsized":"2023-07-14T11:10:07.737919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:56.102535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737624+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:10:07.737624+0000","last_peered":"2023-07-14T11:10:07.737624+0000","last_clean":"2023-07-14T11:10:07.737624+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:10:07.737624+0000","last_undegraded":"2023-07-14T11:10:07.737624+0000","last_fullsized":"2023-07-14T11:10:07.737624+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:49.125150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738992+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:10:07.738992+0000","last_peered":"2023-07-14T11:10:07.738992+0000","last_clean":"2023-07-14T11:10:07.738992+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:10:07.738992+0000","last_undegraded":"2023-07-14T11:10:07.738992+0000","last_fullsized":"2023-07-14T11:10:07.738992+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:49:40.242321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738280+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:10:07.738280+0000","last_peered":"2023-07-14T11:10:07.738280+0000","last_clean":"2023-07-14T11:10:07.738280+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:10:07.738280+0000","last_undegraded":"2023-07-14T11:10:07.738280+0000","last_fullsized":"2023-07-14T11:10:07.738280+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:00:40.108545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141672+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:10:08.141672+0000","last_peered":"2023-07-14T11:10:08.141672+0000","last_clean":"2023-07-14T11:10:08.141672+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:10:08.141672+0000","last_undegraded":"2023-07-14T11:10:08.141672+0000","last_fullsized":"2023-07-14T11:10:08.141672+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:35:25.035847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143146+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:10:08.143146+0000","last_peered":"2023-07-14T11:10:08.143146+0000","last_clean":"2023-07-14T11:10:08.143146+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:10:08.143146+0000","last_undegraded":"2023-07-14T11:10:08.143146+0000","last_fullsized":"2023-07-14T11:10:08.143146+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:00:48.749245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744121+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:10:07.744121+0000","last_peered":"2023-07-14T11:10:07.744121+0000","last_clean":"2023-07-14T11:10:07.744121+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:10:07.744121+0000","last_undegraded":"2023-07-14T11:10:07.744121+0000","last_fullsized":"2023-07-14T11:10:07.744121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:56:30.382410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:27.366892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823702+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:10:07.823702+0000","last_peered":"2023-07-14T11:10:07.823702+0000","last_clean":"2023-07-14T11:10:07.823702+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:10:07.823702+0000","last_undegraded":"2023-07-14T11:10:07.823702+0000","last_fullsized":"2023-07-14T11:10:07.823702+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:42.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739599+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:10:07.739599+0000","last_peered":"2023-07-14T11:10:07.739599+0000","last_clean":"2023-07-14T11:10:07.739599+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:10:07.739599+0000","last_undegraded":"2023-07-14T11:10:07.739599+0000","last_fullsized":"2023-07-14T11:10:07.739599+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:09:31.536272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739315+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:10:07.739315+0000","last_peered":"2023-07-14T11:10:07.739315+0000","last_clean":"2023-07-14T11:10:07.739315+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:10:07.739315+0000","last_undegraded":"2023-07-14T11:10:07.739315+0000","last_fullsized":"2023-07-14T11:10:07.739315+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:06:45.531716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734952+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:10:07.734952+0000","last_peered":"2023-07-14T11:10:07.734952+0000","last_clean":"2023-07-14T11:10:07.734952+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:10:07.734952+0000","last_undegraded":"2023-07-14T11:10:07.734952+0000","last_fullsized":"2023-07-14T11:10:07.734952+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:01.416822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734822+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:10:07.734822+0000","last_peered":"2023-07-14T11:10:07.734822+0000","last_clean":"2023-07-14T11:10:07.734822+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:10:07.734822+0000","last_undegraded":"2023-07-14T11:10:07.734822+0000","last_fullsized":"2023-07-14T11:10:07.734822+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:02:57.300787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737927+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:10:07.737927+0000","last_peered":"2023-07-14T11:10:07.737927+0000","last_clean":"2023-07-14T11:10:07.737927+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:10:07.737927+0000","last_undegraded":"2023-07-14T11:10:07.737927+0000","last_fullsized":"2023-07-14T11:10:07.737927+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:53:58.972191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6118,"num_objects":55,"num_object_clones":0,"num_object_copies":165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55,"num_whiteouts":0,"num_read":11,"num_read_kb":13,"num_write":120,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":37,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":196608,"data_stored":20082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102,"ondisk_log_size":102,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416403,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.65700000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.62}]},{"osd":2,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.5},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.85099999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.70199999999999996}]},{"osd":7,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.82499999999999996}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76600000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.68400000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.75}]}]},{"osd":10,"up_from":72,"seq":309237645333,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28544,"kb_used_data":1592,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716608,"statfs":{"total":95995035648,"available":95965806592,"internally_reserved":0,"allocated":1630208,"data_stored":1083855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.499},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":4.8230000000000004,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.48499999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.38300000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":0.66400000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.4900000000000002,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.51500000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.624},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.64300000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.68300000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":4.8609999999999998,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":0.55000000000000004}]}]},{"osd":9,"up_from":67,"seq":287762808857,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":1003520,"data_stored":489319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.94099999999999995}]},{"osd":1,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.62},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.437}]},{"osd":2,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.498},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.59799999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.76600000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.70299999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.88,"15min":0.88},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.57499999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.9119999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.65100000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"min":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.88500000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.91700000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72999999999999998}]}]},{"osd":8,"up_from":59,"seq":253403070495,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":1019904,"data_stored":489475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76600000000000001,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60499999999999998,"15min":0.61799999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.89000000000000001,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.50800000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64600000000000002,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.45200000000000001,"15min":0.45500000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.63400000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.65200000000000002,"15min":0.66100000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.079,"5min":1.079,"15min":1.079},"last":0.91800000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60299999999999998,"15min":0.60699999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.83099999999999996,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.626,"15min":0.629},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85299999999999998,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.98899999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70599999999999996,"5min":1.052,"15min":1.052},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.74199999999999999,"5min":1.135,"15min":1.135},"last":0.76200000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.69999999999999996,"15min":0.70599999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90600000000000003,"5min":1.008,"15min":1.008},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.72999999999999998,"15min":0.73999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90100000000000002,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":1.0109999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.71199999999999997},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.94399999999999995,"5min":1.028,"15min":1.028},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.72499999999999998,"15min":0.72799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.873,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.95799999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.749,"15min":0.751},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":1.0449999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":1.085}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.117}]}]},{"osd":7,"up_from":51,"seq":219043332131,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28516,"kb_used_data":1564,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716636,"statfs":{"total":95995035648,"available":95965835264,"internally_reserved":0,"allocated":1601536,"data_stored":1079988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60199999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46200000000000002,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.62,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.36199999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65300000000000002,"15min":0.65600000000000003},"min":{"1min":0.45800000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.624,"5min":0.622,"15min":0.622},"min":{"1min":0.47599999999999998,"5min":0.435,"15min":0.435},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.629}]},{"osd":3,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.57699999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59099999999999997,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.56100000000000005}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.76200000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.72399999999999998,"15min":0.72799999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.70999999999999996,"15min":0.71499999999999997},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.878,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.65500000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.745,"15min":0.73899999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.72399999999999998,"15min":0.71999999999999997},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.72399999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.753,"15min":0.75600000000000001},"min":{"1min":0.57499999999999996,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1,"5min":1,"15min":1},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.71299999999999997,"15min":0.71699999999999997},"min":{"1min":0.51900000000000002,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.73699999999999999}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.751},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.70099999999999996}]},{"osd":10,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.71399999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495656,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27928,"kb_used_data":976,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717224,"statfs":{"total":95995035648,"available":95966437376,"internally_reserved":0,"allocated":999424,"data_stored":488737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48799999999999999,"15min":0.46999999999999997},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.628,"15min":0.60999999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95899999999999996,"5min":1.151,"15min":1.151},"last":0.67500000000000004}]},{"osd":1,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.67900000000000005,"15min":0.67500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90200000000000002,"5min":1.407,"15min":1.407},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.57799999999999996}]},{"osd":2,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74199999999999999,"15min":0.72899999999999998},"min":{"1min":0.60299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.995,"5min":1.087,"15min":1.087},"last":0.998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.70399999999999996,"15min":0.70299999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":1.0660000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89000000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.747,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.92200000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.76000000000000001,"15min":0.745},"min":{"1min":0.505,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.091,"5min":1.173,"15min":1.173},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0089999999999999,"5min":1.302,"15min":1.302},"last":0.79700000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.72399999999999998,"15min":0.73899999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86599999999999999,"5min":1.004,"15min":1.004},"last":1.022},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.82699999999999996,"15min":0.81899999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1890000000000001,"5min":1.264,"15min":1.264},"last":1.2150000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.82099999999999995,"15min":0.80100000000000005},"min":{"1min":0.64600000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.83199999999999996,"15min":0.81799999999999995},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":1.157}]},{"osd":8,"last update":"Fri Jul 14 11:09:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.95799999999999996,"15min":0.97499999999999998},"min":{"1min":0.68200000000000005,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.3700000000000001,"5min":1.389,"15min":1.389},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.95099999999999996,"15min":0.97199999999999998},"min":{"1min":0.60899999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1579999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.2010000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.05},"min":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.107}]},{"osd":10,"last update":"Fri Jul 14 11:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.236},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.0900000000000001}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.26}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":492531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.52100000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.90200000000000002,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.54100000000000004,"15min":0.51100000000000001},"min":{"1min":0.35799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Jul 14 11:09:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.58299999999999996,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54300000000000004,"15min":0.51100000000000001},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.72899999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:09:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.56999999999999995,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77700000000000002,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.64600000000000002,"15min":0.58499999999999996},"min":{"1min":0.57099999999999995,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3819999999999997,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.65800000000000003}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.66200000000000003,"15min":0.64200000000000002},"min":{"1min":0.52200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.60899999999999999,"15min":0.59499999999999997},"min":{"1min":0.441,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.0129999999999999,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.69999999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68000000000000005,"15min":0.66300000000000003},"min":{"1min":0.53600000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86799999999999999,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.72999999999999998},"min":{"1min":0.49399999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1180000000000001,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.76700000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.67100000000000004,"15min":0.66000000000000003},"min":{"1min":0.496,"5min":0.316,"15min":0.316},"max":{"1min":0.94999999999999996,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.35199999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77600000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59699999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.66000000000000003,"15min":0.64300000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.745},{"interface":"front","average":{"1min":0.871,"5min":0.73899999999999999,"15min":0.71699999999999997},"min":{"1min":0.51600000000000001,"5min":0.441,"15min":0.441},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.79700000000000004}]},{"osd":9,"last update":"Fri Jul 14 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.67600000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.63300000000000001}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82499999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27936,"kb_used_data":984,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717216,"statfs":{"total":95995035648,"available":95966429184,"internally_reserved":0,"allocated":1007616,"data_stored":489158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58999999999999997,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.94699999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.60399999999999998,"15min":0.55800000000000005},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.1349999999999998,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.51000000000000001}]},{"osd":2,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.56499999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.86099999999999999,"5min":2.528,"15min":2.528},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.61499999999999999,"15min":0.58799999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0840000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Jul 14 11:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.436,"15min":0.42399999999999999},"min":{"1min":0.36699999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.46500000000000002,"15min":0.45700000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42999999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.64500000000000002,"15min":0.61499999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0329999999999999,"5min":2.48,"15min":2.48},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.75,"5min":0.70999999999999996,"15min":0.68300000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.86199999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:42 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.67400000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.68000000000000005,"15min":0.64400000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.028,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":0.84399999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.70599999999999996,"15min":0.69599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99199999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.62},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.71199999999999997,"15min":0.70799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.034,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.66500000000000004}]},{"osd":7,"last update":"Fri Jul 14 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.84199999999999997,"15min":0.84299999999999997},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.79100000000000004,"15min":0.78800000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.77700000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.85099999999999998,"15min":0.86199999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.1799999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.755,"5min":0.86799999999999999,"15min":0.88700000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.1020000000000001,"5min":1.915,"15min":1.915},"last":0.82899999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.88500000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.93100000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.873,"15min":0.873},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.874}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":492292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.49299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.78300000000000003,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.39500000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81299999999999994,"5min":0.879,"15min":0.879},"last":0.68500000000000005}]},{"osd":1,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.54100000000000004,"15min":0.53100000000000003},"min":{"1min":0.41499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77400000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.56200000000000006,"15min":0.54700000000000004},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83099999999999996,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.46899999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.72899999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58599999999999997,"15min":0.56899999999999995},"min":{"1min":0.48199999999999998,"5min":0.377,"15min":0.377},"max":{"1min":0.86799999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.622,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.71299999999999997}]},{"osd":5,"last update":"Fri Jul 14 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.46500000000000002,"15min":0.45600000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.46899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.60899999999999999},"min":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.66200000000000003,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.81100000000000005}]},{"osd":7,"last update":"Fri Jul 14 11:09:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.67200000000000004,"15min":0.66200000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.69999999999999996,"15min":0.69499999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.92200000000000004,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.746}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.57099999999999995,"15min":0.55300000000000005},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.623},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.58699999999999997,"15min":0.57999999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.82699999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.64200000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.84099999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.85799999999999998}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.87}]}]},{"osd":3,"up_from":25,"seq":107374182453,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":1003520,"data_stored":488568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45200000000000001,"15min":0.40999999999999998},"min":{"1min":0.33800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.66200000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47599999999999998,"15min":0.436},"min":{"1min":0.36099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81299999999999994,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.44900000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.38100000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.65800000000000003,"15min":0.59699999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89700000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.85999999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65800000000000003,"15min":0.63300000000000001},"min":{"1min":0.504,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94099999999999995,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.67400000000000004,"15min":0.64600000000000002},"min":{"1min":0.504,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.046,"5min":1.069,"15min":1.069},"last":0.57099999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.65100000000000002,"15min":0.627},"min":{"1min":0.50800000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.71099999999999997,"15min":0.68000000000000005},"min":{"1min":0.56000000000000005,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.93899999999999995}]},{"osd":5,"last update":"Fri Jul 14 11:09:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.751,"15min":0.73699999999999999},"min":{"1min":0.57399999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.024,"5min":1.262,"15min":1.262},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.753,"5min":0.75700000000000001,"15min":0.751},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.151,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.52800000000000002}]},{"osd":6,"last update":"Fri Jul 14 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.745,"15min":0.72799999999999998},"min":{"1min":0.505,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.755,"5min":0.76200000000000001,"15min":0.76300000000000001},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.115,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Fri Jul 14 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.82299999999999995,"15min":0.82899999999999996},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0129999999999999,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.81399999999999995,"15min":0.81000000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.1040000000000001,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.75700000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":1.006}]},{"osd":10,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":3.9199999999999999,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.046}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92200000000000004}]}]},{"osd":4,"up_from":33,"seq":141733920819,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.373,"15min":0.36199999999999999},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.40100000000000002,"15min":0.38500000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.35599999999999998}]},{"osd":2,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80500000000000005,"5min":0.877,"15min":0.877},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.59299999999999997,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89300000000000002,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.54300000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.59899999999999998,"15min":0.57999999999999996},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.59399999999999997,"15min":0.58299999999999996},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.77700000000000002}]},{"osd":5,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.59999999999999998,"15min":0.58199999999999996},"min":{"1min":0.45600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85999999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.65200000000000002,"15min":0.63200000000000001},"min":{"1min":0.505,"5min":0.373,"15min":0.373},"max":{"1min":0.874,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.59399999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.65100000000000002,"15min":0.63500000000000001},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83499999999999996,"5min":1.105,"15min":1.105},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.753,"5min":0.71299999999999997,"15min":0.68100000000000005},"min":{"1min":0.51000000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.90400000000000003,"5min":1.137,"15min":1.137},"last":0.85299999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.63100000000000001,"15min":0.61099999999999999},"min":{"1min":0.51500000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.499},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.60399999999999998,"15min":0.58999999999999997},"min":{"1min":0.44800000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.63100000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.52600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.90900000000000003,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.80400000000000005,"15min":0.79900000000000004},"min":{"1min":0.60299999999999998,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0529999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.75700000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":1},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.73799999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.91600000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.95199999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757293,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28516,"kb_used_data":1564,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716636,"statfs":{"total":95995035648,"available":95965835264,"internally_reserved":0,"allocated":1601536,"data_stored":1079014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73499999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.49099999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68799999999999994,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.77400000000000002,"5min":1.466,"15min":1.466},"last":0.62}]},{"osd":2,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.43099999999999999,"15min":0.42599999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.439,"15min":0.434},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.437}]},{"osd":3,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82499999999999996,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.61299999999999999,"15min":0.61499999999999999},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77100000000000002,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.69799999999999995,"15min":0.65400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":4.9340000000000002,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.67800000000000005,"15min":0.64000000000000001},"min":{"1min":0.40999999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88400000000000001,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68300000000000005,"15min":0.68899999999999995},"min":{"1min":0.51500000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.75}]},{"osd":7,"last update":"Fri Jul 14 11:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.70399999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.90400000000000003,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.73199999999999998,"15min":0.73899999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.90800000000000003,"5min":1.528,"15min":1.528},"last":0.73699999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.629,"15min":0.629},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86099999999999999,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.629,"15min":0.63200000000000001},"min":{"1min":0.371,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83499999999999996,"5min":0.878,"15min":0.878},"last":0.72099999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.755,"5min":0.755,"15min":0.755},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.82499999999999996}]},{"osd":10,"last update":"Fri Jul 14 11:09:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":4.9900000000000002,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.78400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:10:16.803 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-07-14T11:10:17.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 -- 172.21.15.40:0/207637614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e80ffb00 msgr2=0x7f22e80fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:17.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/207637614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e80ffb00 0x7f22e80fff10 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f22d8009990 tx=0x7f22d8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 -- 172.21.15.40:0/207637614 shutdown_connections 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/207637614 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e8101680 0x7f22e8107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/207637614 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e80ffb00 0x7f22e80fff10 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/207637614 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 -- 172.21.15.40:0/207637614 >> 172.21.15.40:0/207637614 conn(0x7f22e80fb0e0 msgr2=0x7f22e80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 -- 172.21.15.40:0/207637614 shutdown_connections 2023-07-14T11:10:17.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.032+0000 7f22ee1ff700 1 -- 172.21.15.40:0/207637614 wait complete. 2023-07-14T11:10:17.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.033+0000 7f22ee1ff700 1 Processor -- start 2023-07-14T11:10:17.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 -- start start 2023-07-14T11:10:17.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e80ffb00 0x7f22e81979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:17.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:17.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 0x7f22e819efa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22e819f570 con 0x7f22e8101680 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22e819f6b0 con 0x7f22e8100cf0 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22ee1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22e819f840 con 0x7f22e80ffb00 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22e6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22e7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 0x7f22e819efa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22e6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38570/0 (socket says 172.21.15.40:38570) 2023-07-14T11:10:17.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22e6ffd700 1 -- 172.21.15.40:0/3634929639 learned_addr learned my addr 172.21.15.40:0/3634929639 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.034+0000 7f22e77fe700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e80ffb00 0x7f22e81979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 -- 172.21.15.40:0/3634929639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e80ffb00 msgr2=0x7f22e81979f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e80ffb00 0x7f22e81979f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 -- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 msgr2=0x7f22e819efa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 0x7f22e819efa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 -- 172.21.15.40:0/3634929639 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22d80095f0 con 0x7f22e8100cf0 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e7fff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 0x7f22e819efa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:17.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e6ffd700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f22dc00e9a0 tx=0x7f22dc00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc009790 con 0x7f22e8100cf0 2023-07-14T11:10:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22dc010430 con 0x7f22e8100cf0 2023-07-14T11:10:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f22e819fb20 con 0x7f22e8100cf0 2023-07-14T11:10:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22dc018680 con 0x7f22e8100cf0 2023-07-14T11:10:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.035+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22e81a0030 con 0x7f22e8100cf0 2023-07-14T11:10:17.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.036+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f22e810b2d0 con 0x7f22e8100cf0 2023-07-14T11:10:17.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.038+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f22dc0105a0 con 0x7f22e8100cf0 2023-07-14T11:10:17.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.040+0000 7f22e4ff9700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 0x7f22d00a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:17.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.040+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f22dc014070 con 0x7f22e8100cf0 2023-07-14T11:10:17.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.040+0000 7f22e77fe700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 0x7f22d00a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:17.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3664247467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2389327160' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:active 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: daemon mds.f assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: Cluster is now healthy 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:active} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: fsmap cephfs:5 2 up:standby-replay 1 up:creating 5 up:standby 4 up:active 2023-07-14T11:10:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:16 smithi130 ceph-mon[129405]: daemon mds.f is now active in filesystem cephfs as rank 4 2023-07-14T11:10:17.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.041+0000 7f22e77fe700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 0x7f22d00a3650 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f22d800b2e0 tx=0x7f22d8005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:17.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3664247467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2389327160' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:active 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: daemon mds.f assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: Cluster is now healthy 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:active} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: fsmap cephfs:5 2 up:standby-replay 1 up:creating 5 up:standby 4 up:active 2023-07-14T11:10:17.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:16 smithi040 ceph-mon[97954]: daemon mds.f is now active in filesystem cephfs as rank 4 2023-07-14T11:10:17.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.044+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f22dc051790 con 0x7f22e8100cf0 2023-07-14T11:10:17.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3664247467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2389327160' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] up:active 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: daemon mds.f assigned to filesystem cephfs as rank 4 (now has 5 ranks) 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: Cluster is now healthy 2023-07-14T11:10:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: fsmap cephfs:4 {0=b=up:active,1=a=up:active,2=c=up:active,3=h=up:active} 2 up:standby-replay 6 up:standby 2023-07-14T11:10:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: fsmap cephfs:5 2 up:standby-replay 1 up:creating 5 up:standby 4 up:active 2023-07-14T11:10:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:16 smithi045 ceph-mon[128791]: daemon mds.f is now active in filesystem cephfs as rank 4 2023-07-14T11:10:17.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.249+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f22e8107c90 con 0x7f22d00a11a0 2023-07-14T11:10:17.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.262+0000 7f22e4ff9700 1 -- 172.21.15.40:0/3634929639 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+422290 (secure 0 0 0) 0x7f22e8107c90 con 0x7f22d00a11a0 2023-07-14T11:10:17.268 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:17.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 msgr2=0x7f22d00a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:17.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 0x7f22d00a3650 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f22d800b2e0 tx=0x7f22d8005960 comp rx=0 tx=0).stop 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 msgr2=0x7f22e8197f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f22dc00e9a0 tx=0x7f22dc00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 shutdown_connections 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22d00a11a0 0x7f22d00a3650 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22e8101680 0x7f22e819efa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22e80ffb00 0x7f22e81979f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 --2- 172.21.15.40:0/3634929639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22e8100cf0 0x7f22e8197f30 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:17.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 >> 172.21.15.40:0/3634929639 conn(0x7f22e80fb0e0 msgr2=0x7f22e8106570 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:17.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 shutdown_connections 2023-07-14T11:10:17.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:17.267+0000 7f22ee1ff700 1 -- 172.21.15.40:0/3634929639 wait complete. 2023-07-14T11:10:17.272 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:10:17.287 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":295,"stamp":"2023-07-14T11:10:16.632543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":596486,"num_objects":57,"num_object_clones":0,"num_object_copies":171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":57,"num_whiteouts":0,"num_read":147,"num_read_kb":132,"num_write":301,"num_write_kb":2979,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":37,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":239,"ondisk_log_size":239,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337012,"kb_used_data":13588,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604812,"statfs":{"total":1151940427776,"available":1151595327488,"internally_reserved":0,"allocated":13914112,"data_stored":7651900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1667,"num_objects":33,"num_object_clones":0,"num_object_copies":99,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33,"num_whiteouts":0,"num_read":11,"num_read_kb":13,"num_write":72,"num_write_kb":23,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":23,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.902015"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738359+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:10:07.738359+0000","last_peered":"2023-07-14T11:10:07.738359+0000","last_clean":"2023-07-14T11:10:07.738359+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:10:07.738359+0000","last_undegraded":"2023-07-14T11:10:07.738359+0000","last_fullsized":"2023-07-14T11:10:07.738359+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:21:10.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141620+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:10:08.141620+0000","last_peered":"2023-07-14T11:10:08.141620+0000","last_clean":"2023-07-14T11:10:08.141620+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:10:08.141620+0000","last_undegraded":"2023-07-14T11:10:08.141620+0000","last_fullsized":"2023-07-14T11:10:08.141620+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:43:13.354815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142202+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:10:08.142202+0000","last_peered":"2023-07-14T11:10:08.142202+0000","last_clean":"2023-07-14T11:10:08.142202+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:10:08.142202+0000","last_undegraded":"2023-07-14T11:10:08.142202+0000","last_fullsized":"2023-07-14T11:10:08.142202+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:25:21.721235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823979+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:10:07.823979+0000","last_peered":"2023-07-14T11:10:07.823979+0000","last_clean":"2023-07-14T11:10:07.823979+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:10:07.823979+0000","last_undegraded":"2023-07-14T11:10:07.823979+0000","last_fullsized":"2023-07-14T11:10:07.823979+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:50:39.156238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734553+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:10:07.734553+0000","last_peered":"2023-07-14T11:10:07.734553+0000","last_clean":"2023-07-14T11:10:07.734553+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:10:07.734553+0000","last_undegraded":"2023-07-14T11:10:07.734553+0000","last_fullsized":"2023-07-14T11:10:07.734553+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:08:20.010349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142872+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:10:08.142872+0000","last_peered":"2023-07-14T11:10:08.142872+0000","last_clean":"2023-07-14T11:10:08.142872+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:10:08.142872+0000","last_undegraded":"2023-07-14T11:10:08.142872+0000","last_fullsized":"2023-07-14T11:10:08.142872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:38:58.846023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736546+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:10:07.736546+0000","last_peered":"2023-07-14T11:10:07.736546+0000","last_clean":"2023-07-14T11:10:07.736546+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:10:07.736546+0000","last_undegraded":"2023-07-14T11:10:07.736546+0000","last_fullsized":"2023-07-14T11:10:07.736546+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:29:36.124497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739054+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:10:07.739054+0000","last_peered":"2023-07-14T11:10:07.739054+0000","last_clean":"2023-07-14T11:10:07.739054+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:10:07.739054+0000","last_undegraded":"2023-07-14T11:10:07.739054+0000","last_fullsized":"2023-07-14T11:10:07.739054+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:48:52.238878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735166+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:10:07.735166+0000","last_peered":"2023-07-14T11:10:07.735166+0000","last_clean":"2023-07-14T11:10:07.735166+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:10:07.735166+0000","last_undegraded":"2023-07-14T11:10:07.735166+0000","last_fullsized":"2023-07-14T11:10:07.735166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:45.770708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143166+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:10:08.143166+0000","last_peered":"2023-07-14T11:10:08.143166+0000","last_clean":"2023-07-14T11:10:08.143166+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:10:08.143166+0000","last_undegraded":"2023-07-14T11:10:08.143166+0000","last_fullsized":"2023-07-14T11:10:08.143166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:11.980357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737468+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:10:07.737468+0000","last_peered":"2023-07-14T11:10:07.737468+0000","last_clean":"2023-07-14T11:10:07.737468+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:10:07.737468+0000","last_undegraded":"2023-07-14T11:10:07.737468+0000","last_fullsized":"2023-07-14T11:10:07.737468+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:31:39.003266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823717+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:10:07.823717+0000","last_peered":"2023-07-14T11:10:07.823717+0000","last_clean":"2023-07-14T11:10:07.823717+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:10:07.823717+0000","last_undegraded":"2023-07-14T11:10:07.823717+0000","last_fullsized":"2023-07-14T11:10:07.823717+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:03:29.844122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734676+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:10:07.734676+0000","last_peered":"2023-07-14T11:10:07.734676+0000","last_clean":"2023-07-14T11:10:07.734676+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:10:07.734676+0000","last_undegraded":"2023-07-14T11:10:07.734676+0000","last_fullsized":"2023-07-14T11:10:07.734676+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:10:43.308729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141270+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:10:08.141270+0000","last_peered":"2023-07-14T11:10:08.141270+0000","last_clean":"2023-07-14T11:10:08.141270+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:10:08.141270+0000","last_undegraded":"2023-07-14T11:10:08.141270+0000","last_fullsized":"2023-07-14T11:10:08.141270+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:20:00.054130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141301+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:10:08.141301+0000","last_peered":"2023-07-14T11:10:08.141301+0000","last_clean":"2023-07-14T11:10:08.141301+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:10:08.141301+0000","last_undegraded":"2023-07-14T11:10:08.141301+0000","last_fullsized":"2023-07-14T11:10:08.141301+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:38:22.587911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734282+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:10:07.734282+0000","last_peered":"2023-07-14T11:10:07.734282+0000","last_clean":"2023-07-14T11:10:07.734282+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:10:07.734282+0000","last_undegraded":"2023-07-14T11:10:07.734282+0000","last_fullsized":"2023-07-14T11:10:07.734282+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:53:57.650717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738135+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:10:07.738135+0000","last_peered":"2023-07-14T11:10:07.738135+0000","last_clean":"2023-07-14T11:10:07.738135+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:10:07.738135+0000","last_undegraded":"2023-07-14T11:10:07.738135+0000","last_fullsized":"2023-07-14T11:10:07.738135+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:48:17.654653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740775+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:10:07.740775+0000","last_peered":"2023-07-14T11:10:07.740775+0000","last_clean":"2023-07-14T11:10:07.740775+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:10:07.740775+0000","last_undegraded":"2023-07-14T11:10:07.740775+0000","last_fullsized":"2023-07-14T11:10:07.740775+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:36:51.637622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737923+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:10:07.737923+0000","last_peered":"2023-07-14T11:10:07.737923+0000","last_clean":"2023-07-14T11:10:07.737923+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:07.737923+0000","last_undegraded":"2023-07-14T11:10:07.737923+0000","last_fullsized":"2023-07-14T11:10:07.737923+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:45.243063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738752+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:10:07.738752+0000","last_peered":"2023-07-14T11:10:07.738752+0000","last_clean":"2023-07-14T11:10:07.738752+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:10:07.738752+0000","last_undegraded":"2023-07-14T11:10:07.738752+0000","last_fullsized":"2023-07-14T11:10:07.738752+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:12:05.905433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"83'4","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740661+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:10:07.740661+0000","last_peered":"2023-07-14T11:10:07.740661+0000","last_clean":"2023-07-14T11:10:07.740661+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:10:07.740661+0000","last_undegraded":"2023-07-14T11:10:07.740661+0000","last_fullsized":"2023-07-14T11:10:07.740661+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:21:25.983167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:53.251986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822942+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:10:07.822942+0000","last_peered":"2023-07-14T11:10:07.822942+0000","last_clean":"2023-07-14T11:10:07.822942+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:10:07.822942+0000","last_undegraded":"2023-07-14T11:10:07.822942+0000","last_fullsized":"2023-07-14T11:10:07.822942+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:32:00.530574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740281+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:10:07.740281+0000","last_peered":"2023-07-14T11:10:07.740281+0000","last_clean":"2023-07-14T11:10:07.740281+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:10:07.740281+0000","last_undegraded":"2023-07-14T11:10:07.740281+0000","last_fullsized":"2023-07-14T11:10:07.740281+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:56.375572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142002+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:10:08.142002+0000","last_peered":"2023-07-14T11:10:08.142002+0000","last_clean":"2023-07-14T11:10:08.142002+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:10:08.142002+0000","last_undegraded":"2023-07-14T11:10:08.142002+0000","last_fullsized":"2023-07-14T11:10:08.142002+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:26.963699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823419+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:10:07.823419+0000","last_peered":"2023-07-14T11:10:07.823419+0000","last_clean":"2023-07-14T11:10:07.823419+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:10:07.823419+0000","last_undegraded":"2023-07-14T11:10:07.823419+0000","last_fullsized":"2023-07-14T11:10:07.823419+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:43.101537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'11","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375838+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:10:16.375838+0000","last_peered":"2023-07-14T11:10:16.375838+0000","last_clean":"2023-07-14T11:10:16.375838+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:10:16.375838+0000","last_undegraded":"2023-07-14T11:10:16.375838+0000","last_fullsized":"2023-07-14T11:10:16.375838+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":11,"log_dups_size":0,"ondisk_log_size":11,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:26:07.506765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734736+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:10:07.734736+0000","last_peered":"2023-07-14T11:10:07.734736+0000","last_clean":"2023-07-14T11:10:07.734736+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:10:07.734736+0000","last_undegraded":"2023-07-14T11:10:07.734736+0000","last_fullsized":"2023-07-14T11:10:07.734736+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:10:30.512188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822955+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:10:07.822955+0000","last_peered":"2023-07-14T11:10:07.822955+0000","last_clean":"2023-07-14T11:10:07.822955+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:10:07.822955+0000","last_undegraded":"2023-07-14T11:10:07.822955+0000","last_fullsized":"2023-07-14T11:10:07.822955+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:44.974006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376472+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:10:16.376472+0000","last_peered":"2023-07-14T11:10:16.376472+0000","last_clean":"2023-07-14T11:10:16.376472+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:10:16.376472+0000","last_undegraded":"2023-07-14T11:10:16.376472+0000","last_fullsized":"2023-07-14T11:10:16.376472+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:41:26.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740246+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:10:07.740246+0000","last_peered":"2023-07-14T11:10:07.740246+0000","last_clean":"2023-07-14T11:10:07.740246+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:10:07.740246+0000","last_undegraded":"2023-07-14T11:10:07.740246+0000","last_fullsized":"2023-07-14T11:10:07.740246+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:32.963491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738142+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:10:07.738142+0000","last_peered":"2023-07-14T11:10:07.738142+0000","last_clean":"2023-07-14T11:10:07.738142+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:10:07.738142+0000","last_undegraded":"2023-07-14T11:10:07.738142+0000","last_fullsized":"2023-07-14T11:10:07.738142+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:51:58.731259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739304+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:10:07.739304+0000","last_peered":"2023-07-14T11:10:07.739304+0000","last_clean":"2023-07-14T11:10:07.739304+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:10:07.739304+0000","last_undegraded":"2023-07-14T11:10:07.739304+0000","last_fullsized":"2023-07-14T11:10:07.739304+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:06:48.309845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734873+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:10:07.734873+0000","last_peered":"2023-07-14T11:10:07.734873+0000","last_clean":"2023-07-14T11:10:07.734873+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:10:07.734873+0000","last_undegraded":"2023-07-14T11:10:07.734873+0000","last_fullsized":"2023-07-14T11:10:07.734873+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:48.090320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738348+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:10:07.738348+0000","last_peered":"2023-07-14T11:10:07.738348+0000","last_clean":"2023-07-14T11:10:07.738348+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:10:07.738348+0000","last_undegraded":"2023-07-14T11:10:07.738348+0000","last_fullsized":"2023-07-14T11:10:07.738348+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:36:58.104207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357621+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:10:14.357621+0000","last_peered":"2023-07-14T11:10:14.357621+0000","last_clean":"2023-07-14T11:10:14.357621+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:10:14.357621+0000","last_undegraded":"2023-07-14T11:10:14.357621+0000","last_fullsized":"2023-07-14T11:10:14.357621+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:38:46.335485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.745285+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:10:07.745285+0000","last_peered":"2023-07-14T11:10:07.745285+0000","last_clean":"2023-07-14T11:10:07.745285+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:10:07.745285+0000","last_undegraded":"2023-07-14T11:10:07.745285+0000","last_fullsized":"2023-07-14T11:10:07.745285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:17.289951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737973+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:10:07.737973+0000","last_peered":"2023-07-14T11:10:07.737973+0000","last_clean":"2023-07-14T11:10:07.737973+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:10:07.737973+0000","last_undegraded":"2023-07-14T11:10:07.737973+0000","last_fullsized":"2023-07-14T11:10:07.737973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:07:16.888150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734417+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:10:07.734417+0000","last_peered":"2023-07-14T11:10:07.734417+0000","last_clean":"2023-07-14T11:10:07.734417+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:10:07.734417+0000","last_undegraded":"2023-07-14T11:10:07.734417+0000","last_fullsized":"2023-07-14T11:10:07.734417+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:53:35.630621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738872+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:10:07.738872+0000","last_peered":"2023-07-14T11:10:07.738872+0000","last_clean":"2023-07-14T11:10:07.738872+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:10:07.738872+0000","last_undegraded":"2023-07-14T11:10:07.738872+0000","last_fullsized":"2023-07-14T11:10:07.738872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:01:08.293055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141456+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:10:08.141456+0000","last_peered":"2023-07-14T11:10:08.141456+0000","last_clean":"2023-07-14T11:10:08.141456+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:10:08.141456+0000","last_undegraded":"2023-07-14T11:10:08.141456+0000","last_fullsized":"2023-07-14T11:10:08.141456+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:14:59.191235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739475+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:10:07.739475+0000","last_peered":"2023-07-14T11:10:07.739475+0000","last_clean":"2023-07-14T11:10:07.739475+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:10:07.739475+0000","last_undegraded":"2023-07-14T11:10:07.739475+0000","last_fullsized":"2023-07-14T11:10:07.739475+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:39:15.619767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738091+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:10:07.738091+0000","last_peered":"2023-07-14T11:10:07.738091+0000","last_clean":"2023-07-14T11:10:07.738091+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:10:07.738091+0000","last_undegraded":"2023-07-14T11:10:07.738091+0000","last_fullsized":"2023-07-14T11:10:07.738091+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:48:41.840080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739132+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:10:07.739132+0000","last_peered":"2023-07-14T11:10:07.739132+0000","last_clean":"2023-07-14T11:10:07.739132+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:10:07.739132+0000","last_undegraded":"2023-07-14T11:10:07.739132+0000","last_fullsized":"2023-07-14T11:10:07.739132+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:39:26.932728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783768+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:10:13.783768+0000","last_peered":"2023-07-14T11:10:13.783768+0000","last_clean":"2023-07-14T11:10:13.783768+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:10:13.783768+0000","last_undegraded":"2023-07-14T11:10:13.783768+0000","last_fullsized":"2023-07-14T11:10:13.783768+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:10:05.049869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735216+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:10:07.735216+0000","last_peered":"2023-07-14T11:10:07.735216+0000","last_clean":"2023-07-14T11:10:07.735216+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:10:07.735216+0000","last_undegraded":"2023-07-14T11:10:07.735216+0000","last_fullsized":"2023-07-14T11:10:07.735216+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:50.720818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737634+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:10:07.737634+0000","last_peered":"2023-07-14T11:10:07.737634+0000","last_clean":"2023-07-14T11:10:07.737634+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:10:07.737634+0000","last_undegraded":"2023-07-14T11:10:07.737634+0000","last_fullsized":"2023-07-14T11:10:07.737634+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:25:31.401998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783874+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:10:13.783874+0000","last_peered":"2023-07-14T11:10:13.783874+0000","last_clean":"2023-07-14T11:10:13.783874+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:10:13.783874+0000","last_undegraded":"2023-07-14T11:10:13.783874+0000","last_fullsized":"2023-07-14T11:10:13.783874+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:10:40.132095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142089+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:10:08.142089+0000","last_peered":"2023-07-14T11:10:08.142089+0000","last_clean":"2023-07-14T11:10:08.142089+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:10:08.142089+0000","last_undegraded":"2023-07-14T11:10:08.142089+0000","last_fullsized":"2023-07-14T11:10:08.142089+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:56:07.808822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823945+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:10:07.823945+0000","last_peered":"2023-07-14T11:10:07.823945+0000","last_clean":"2023-07-14T11:10:07.823945+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:10:07.823945+0000","last_undegraded":"2023-07-14T11:10:07.823945+0000","last_fullsized":"2023-07-14T11:10:07.823945+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:16:13.828703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"83'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740105+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:10:07.740105+0000","last_peered":"2023-07-14T11:10:07.740105+0000","last_clean":"2023-07-14T11:10:07.740105+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:10:07.740105+0000","last_undegraded":"2023-07-14T11:10:07.740105+0000","last_fullsized":"2023-07-14T11:10:07.740105+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:30:17.168629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738962+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:10:07.738962+0000","last_peered":"2023-07-14T11:10:07.738962+0000","last_clean":"2023-07-14T11:10:07.738962+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:10:07.738962+0000","last_undegraded":"2023-07-14T11:10:07.738962+0000","last_fullsized":"2023-07-14T11:10:07.738962+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:25:22.187958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737948+0000","last_change":"2023-07-14T11:09:47.624060+0000","last_active":"2023-07-14T11:10:07.737948+0000","last_peered":"2023-07-14T11:10:07.737948+0000","last_clean":"2023-07-14T11:10:07.737948+0000","last_became_active":"2023-07-14T11:09:47.623605+0000","last_became_peered":"2023-07-14T11:09:47.623605+0000","last_unstale":"2023-07-14T11:10:07.737948+0000","last_undegraded":"2023-07-14T11:10:07.737948+0000","last_fullsized":"2023-07-14T11:10:07.737948+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:17:15.881693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"90'12","reported_seq":42,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.352417+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:10:14.352417+0000","last_peered":"2023-07-14T11:10:14.352417+0000","last_clean":"2023-07-14T11:10:14.352417+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:10:14.352417+0000","last_undegraded":"2023-07-14T11:10:14.352417+0000","last_fullsized":"2023-07-14T11:10:14.352417+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:01:52.717646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3794,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":6,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734684+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:10:07.734684+0000","last_peered":"2023-07-14T11:10:07.734684+0000","last_clean":"2023-07-14T11:10:07.734684+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:10:07.734684+0000","last_undegraded":"2023-07-14T11:10:07.734684+0000","last_fullsized":"2023-07-14T11:10:07.734684+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:55.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740741+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:10:07.740741+0000","last_peered":"2023-07-14T11:10:07.740741+0000","last_clean":"2023-07-14T11:10:07.740741+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:10:07.740741+0000","last_undegraded":"2023-07-14T11:10:07.740741+0000","last_fullsized":"2023-07-14T11:10:07.740741+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:37.113063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"83'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737743+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:10:07.737743+0000","last_peered":"2023-07-14T11:10:07.737743+0000","last_clean":"2023-07-14T11:10:07.737743+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:10:07.737743+0000","last_undegraded":"2023-07-14T11:10:07.737743+0000","last_fullsized":"2023-07-14T11:10:07.737743+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:39:38.915458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734456+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:10:07.734456+0000","last_peered":"2023-07-14T11:10:07.734456+0000","last_clean":"2023-07-14T11:10:07.734456+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:10:07.734456+0000","last_undegraded":"2023-07-14T11:10:07.734456+0000","last_fullsized":"2023-07-14T11:10:07.734456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:25:56.604869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744522+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:10:07.744522+0000","last_peered":"2023-07-14T11:10:07.744522+0000","last_clean":"2023-07-14T11:10:07.744522+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:10:07.744522+0000","last_undegraded":"2023-07-14T11:10:07.744522+0000","last_fullsized":"2023-07-14T11:10:07.744522+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:08.719446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741523+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:10:07.741523+0000","last_peered":"2023-07-14T11:10:07.741523+0000","last_clean":"2023-07-14T11:10:07.741523+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:10:07.741523+0000","last_undegraded":"2023-07-14T11:10:07.741523+0000","last_fullsized":"2023-07-14T11:10:07.741523+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:12:45.811503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736758+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:10:07.736758+0000","last_peered":"2023-07-14T11:10:07.736758+0000","last_clean":"2023-07-14T11:10:07.736758+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:10:07.736758+0000","last_undegraded":"2023-07-14T11:10:07.736758+0000","last_fullsized":"2023-07-14T11:10:07.736758+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:28.882720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740995+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:10:07.740995+0000","last_peered":"2023-07-14T11:10:07.740995+0000","last_clean":"2023-07-14T11:10:07.740995+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:10:07.740995+0000","last_undegraded":"2023-07-14T11:10:07.740995+0000","last_fullsized":"2023-07-14T11:10:07.740995+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:35.770995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.368124+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:10:15.368124+0000","last_peered":"2023-07-14T11:10:15.368124+0000","last_clean":"2023-07-14T11:10:15.368124+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:10:15.368124+0000","last_undegraded":"2023-07-14T11:10:15.368124+0000","last_fullsized":"2023-07-14T11:10:15.368124+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:57:32.688631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737387+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:10:07.737387+0000","last_peered":"2023-07-14T11:10:07.737387+0000","last_clean":"2023-07-14T11:10:07.737387+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:10:07.737387+0000","last_undegraded":"2023-07-14T11:10:07.737387+0000","last_fullsized":"2023-07-14T11:10:07.737387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:18:06.626041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734847+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:10:07.734847+0000","last_peered":"2023-07-14T11:10:07.734847+0000","last_clean":"2023-07-14T11:10:07.734847+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:10:07.734847+0000","last_undegraded":"2023-07-14T11:10:07.734847+0000","last_fullsized":"2023-07-14T11:10:07.734847+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:14:05.500061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":43,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369490+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:10:15.369490+0000","last_peered":"2023-07-14T11:10:15.369490+0000","last_clean":"2023-07-14T11:10:15.369490+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:10:15.369490+0000","last_undegraded":"2023-07-14T11:10:15.369490+0000","last_fullsized":"2023-07-14T11:10:15.369490+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:26:26.993318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741882+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:10:07.741882+0000","last_peered":"2023-07-14T11:10:07.741882+0000","last_clean":"2023-07-14T11:10:07.741882+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:10:07.741882+0000","last_undegraded":"2023-07-14T11:10:07.741882+0000","last_fullsized":"2023-07-14T11:10:07.741882+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:38:12.515688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741815+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:10:07.741815+0000","last_peered":"2023-07-14T11:10:07.741815+0000","last_clean":"2023-07-14T11:10:07.741815+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:10:07.741815+0000","last_undegraded":"2023-07-14T11:10:07.741815+0000","last_fullsized":"2023-07-14T11:10:07.741815+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:39.325247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"83'1","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734769+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:10:07.734769+0000","last_peered":"2023-07-14T11:10:07.734769+0000","last_clean":"2023-07-14T11:10:07.734769+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:10:07.734769+0000","last_undegraded":"2023-07-14T11:10:07.734769+0000","last_fullsized":"2023-07-14T11:10:07.734769+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:16.215502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738122+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:10:07.738122+0000","last_peered":"2023-07-14T11:10:07.738122+0000","last_clean":"2023-07-14T11:10:07.738122+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:10:07.738122+0000","last_undegraded":"2023-07-14T11:10:07.738122+0000","last_fullsized":"2023-07-14T11:10:07.738122+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:28.226381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734949+0000","last_change":"2023-07-14T11:09:47.736950+0000","last_active":"2023-07-14T11:10:07.734949+0000","last_peered":"2023-07-14T11:10:07.734949+0000","last_clean":"2023-07-14T11:10:07.734949+0000","last_became_active":"2023-07-14T11:09:47.736506+0000","last_became_peered":"2023-07-14T11:09:47.736506+0000","last_unstale":"2023-07-14T11:10:07.734949+0000","last_undegraded":"2023-07-14T11:10:07.734949+0000","last_fullsized":"2023-07-14T11:10:07.734949+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:24.567987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738616+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:10:07.738616+0000","last_peered":"2023-07-14T11:10:07.738616+0000","last_clean":"2023-07-14T11:10:07.738616+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:10:07.738616+0000","last_undegraded":"2023-07-14T11:10:07.738616+0000","last_fullsized":"2023-07-14T11:10:07.738616+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:11:19.303557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142811+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:10:08.142811+0000","last_peered":"2023-07-14T11:10:08.142811+0000","last_clean":"2023-07-14T11:10:08.142811+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:08.142811+0000","last_undegraded":"2023-07-14T11:10:08.142811+0000","last_fullsized":"2023-07-14T11:10:08.142811+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:51:22.856363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735220+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:10:07.735220+0000","last_peered":"2023-07-14T11:10:07.735220+0000","last_clean":"2023-07-14T11:10:07.735220+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:10:07.735220+0000","last_undegraded":"2023-07-14T11:10:07.735220+0000","last_fullsized":"2023-07-14T11:10:07.735220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:52:35.457076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736779+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:10:07.736779+0000","last_peered":"2023-07-14T11:10:07.736779+0000","last_clean":"2023-07-14T11:10:07.736779+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:10:07.736779+0000","last_undegraded":"2023-07-14T11:10:07.736779+0000","last_fullsized":"2023-07-14T11:10:07.736779+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:49:44.032115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143050+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:10:08.143050+0000","last_peered":"2023-07-14T11:10:08.143050+0000","last_clean":"2023-07-14T11:10:08.143050+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:10:08.143050+0000","last_undegraded":"2023-07-14T11:10:08.143050+0000","last_fullsized":"2023-07-14T11:10:08.143050+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:04:11.876243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734819+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:10:07.734819+0000","last_peered":"2023-07-14T11:10:07.734819+0000","last_clean":"2023-07-14T11:10:07.734819+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:10:07.734819+0000","last_undegraded":"2023-07-14T11:10:07.734819+0000","last_fullsized":"2023-07-14T11:10:07.734819+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:26:53.174066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739718+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:10:07.739718+0000","last_peered":"2023-07-14T11:10:07.739718+0000","last_clean":"2023-07-14T11:10:07.739718+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:10:07.739718+0000","last_undegraded":"2023-07-14T11:10:07.739718+0000","last_fullsized":"2023-07-14T11:10:07.739718+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:16:20.330278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734207+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:10:07.734207+0000","last_peered":"2023-07-14T11:10:07.734207+0000","last_clean":"2023-07-14T11:10:07.734207+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:10:07.734207+0000","last_undegraded":"2023-07-14T11:10:07.734207+0000","last_fullsized":"2023-07-14T11:10:07.734207+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:50:46.452985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736543+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:10:07.736543+0000","last_peered":"2023-07-14T11:10:07.736543+0000","last_clean":"2023-07-14T11:10:07.736543+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:10:07.736543+0000","last_undegraded":"2023-07-14T11:10:07.736543+0000","last_fullsized":"2023-07-14T11:10:07.736543+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:56:32.791400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738925+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:10:07.738925+0000","last_peered":"2023-07-14T11:10:07.738925+0000","last_clean":"2023-07-14T11:10:07.738925+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:10:07.738925+0000","last_undegraded":"2023-07-14T11:10:07.738925+0000","last_fullsized":"2023-07-14T11:10:07.738925+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:02:28.884240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.367881+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:10:15.367881+0000","last_peered":"2023-07-14T11:10:15.367881+0000","last_clean":"2023-07-14T11:10:15.367881+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:10:15.367881+0000","last_undegraded":"2023-07-14T11:10:15.367881+0000","last_fullsized":"2023-07-14T11:10:15.367881+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:12:29.230412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":354,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736416+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:10:07.736416+0000","last_peered":"2023-07-14T11:10:07.736416+0000","last_clean":"2023-07-14T11:10:07.736416+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:10:07.736416+0000","last_undegraded":"2023-07-14T11:10:07.736416+0000","last_fullsized":"2023-07-14T11:10:07.736416+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:40:53.088449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741053+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:10:07.741053+0000","last_peered":"2023-07-14T11:10:07.741053+0000","last_clean":"2023-07-14T11:10:07.741053+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:10:07.741053+0000","last_undegraded":"2023-07-14T11:10:07.741053+0000","last_fullsized":"2023-07-14T11:10:07.741053+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:46:21.793891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734132+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:10:07.734132+0000","last_peered":"2023-07-14T11:10:07.734132+0000","last_clean":"2023-07-14T11:10:07.734132+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:10:07.734132+0000","last_undegraded":"2023-07-14T11:10:07.734132+0000","last_fullsized":"2023-07-14T11:10:07.734132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:58:44.611438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'3","reported_seq":35,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357053+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:10:14.357053+0000","last_peered":"2023-07-14T11:10:14.357053+0000","last_clean":"2023-07-14T11:10:14.357053+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:10:14.357053+0000","last_undegraded":"2023-07-14T11:10:14.357053+0000","last_fullsized":"2023-07-14T11:10:14.357053+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:06:23.046904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737806+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:10:07.737806+0000","last_peered":"2023-07-14T11:10:07.737806+0000","last_clean":"2023-07-14T11:10:07.737806+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:10:07.737806+0000","last_undegraded":"2023-07-14T11:10:07.737806+0000","last_fullsized":"2023-07-14T11:10:07.737806+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:15.765387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.370106+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:10:15.370106+0000","last_peered":"2023-07-14T11:10:15.370106+0000","last_clean":"2023-07-14T11:10:15.370106+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:10:15.370106+0000","last_undegraded":"2023-07-14T11:10:15.370106+0000","last_fullsized":"2023-07-14T11:10:15.370106+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:27:03.761811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734605+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:10:07.734605+0000","last_peered":"2023-07-14T11:10:07.734605+0000","last_clean":"2023-07-14T11:10:07.734605+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:10:07.734605+0000","last_undegraded":"2023-07-14T11:10:07.734605+0000","last_fullsized":"2023-07-14T11:10:07.734605+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:17:38.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:57.136733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"87'1","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.748353+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:10:07.748353+0000","last_peered":"2023-07-14T11:10:07.748353+0000","last_clean":"2023-07-14T11:10:07.748353+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:10:07.748353+0000","last_undegraded":"2023-07-14T11:10:07.748353+0000","last_fullsized":"2023-07-14T11:10:07.748353+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:47:01.844602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740024+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:10:07.740024+0000","last_peered":"2023-07-14T11:10:07.740024+0000","last_clean":"2023-07-14T11:10:07.740024+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:10:07.740024+0000","last_undegraded":"2023-07-14T11:10:07.740024+0000","last_fullsized":"2023-07-14T11:10:07.740024+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:34:33.255530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738545+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:10:07.738545+0000","last_peered":"2023-07-14T11:10:07.738545+0000","last_clean":"2023-07-14T11:10:07.738545+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:10:07.738545+0000","last_undegraded":"2023-07-14T11:10:07.738545+0000","last_fullsized":"2023-07-14T11:10:07.738545+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:36.240727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375131+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:10:16.375131+0000","last_peered":"2023-07-14T11:10:16.375131+0000","last_clean":"2023-07-14T11:10:16.375131+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:10:16.375131+0000","last_undegraded":"2023-07-14T11:10:16.375131+0000","last_fullsized":"2023-07-14T11:10:16.375131+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:02:18.139490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141864+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:10:08.141864+0000","last_peered":"2023-07-14T11:10:08.141864+0000","last_clean":"2023-07-14T11:10:08.141864+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:10:08.141864+0000","last_undegraded":"2023-07-14T11:10:08.141864+0000","last_fullsized":"2023-07-14T11:10:08.141864+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:28:33.937520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823662+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:10:07.823662+0000","last_peered":"2023-07-14T11:10:07.823662+0000","last_clean":"2023-07-14T11:10:07.823662+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:10:07.823662+0000","last_undegraded":"2023-07-14T11:10:07.823662+0000","last_fullsized":"2023-07-14T11:10:07.823662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:28:39.306429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.782018+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:10:13.782018+0000","last_peered":"2023-07-14T11:10:13.782018+0000","last_clean":"2023-07-14T11:10:13.782018+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:10:13.782018+0000","last_undegraded":"2023-07-14T11:10:13.782018+0000","last_fullsized":"2023-07-14T11:10:13.782018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:50:14.638370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734307+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:10:07.734307+0000","last_peered":"2023-07-14T11:10:07.734307+0000","last_clean":"2023-07-14T11:10:07.734307+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:10:07.734307+0000","last_undegraded":"2023-07-14T11:10:07.734307+0000","last_fullsized":"2023-07-14T11:10:07.734307+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:27:40.728690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740731+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:10:07.740731+0000","last_peered":"2023-07-14T11:10:07.740731+0000","last_clean":"2023-07-14T11:10:07.740731+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:10:07.740731+0000","last_undegraded":"2023-07-14T11:10:07.740731+0000","last_fullsized":"2023-07-14T11:10:07.740731+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:37.925348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739281+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:10:07.739281+0000","last_peered":"2023-07-14T11:10:07.739281+0000","last_clean":"2023-07-14T11:10:07.739281+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:10:07.739281+0000","last_undegraded":"2023-07-14T11:10:07.739281+0000","last_fullsized":"2023-07-14T11:10:07.739281+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:35:40.777893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737586+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:10:07.737586+0000","last_peered":"2023-07-14T11:10:07.737586+0000","last_clean":"2023-07-14T11:10:07.737586+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:10:07.737586+0000","last_undegraded":"2023-07-14T11:10:07.737586+0000","last_fullsized":"2023-07-14T11:10:07.737586+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:43.286237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823509+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:10:07.823509+0000","last_peered":"2023-07-14T11:10:07.823509+0000","last_clean":"2023-07-14T11:10:07.823509+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:10:07.823509+0000","last_undegraded":"2023-07-14T11:10:07.823509+0000","last_fullsized":"2023-07-14T11:10:07.823509+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:28:35.378022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738408+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:10:07.738408+0000","last_peered":"2023-07-14T11:10:07.738408+0000","last_clean":"2023-07-14T11:10:07.738408+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:10:07.738408+0000","last_undegraded":"2023-07-14T11:10:07.738408+0000","last_fullsized":"2023-07-14T11:10:07.738408+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:10.923637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739938+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:10:07.739938+0000","last_peered":"2023-07-14T11:10:07.739938+0000","last_clean":"2023-07-14T11:10:07.739938+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:10:07.739938+0000","last_undegraded":"2023-07-14T11:10:07.739938+0000","last_fullsized":"2023-07-14T11:10:07.739938+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:27:36.946955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823764+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:10:07.823764+0000","last_peered":"2023-07-14T11:10:07.823764+0000","last_clean":"2023-07-14T11:10:07.823764+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:10:07.823764+0000","last_undegraded":"2023-07-14T11:10:07.823764+0000","last_fullsized":"2023-07-14T11:10:07.823764+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:46:25.967420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783411+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:10:13.783411+0000","last_peered":"2023-07-14T11:10:13.783411+0000","last_clean":"2023-07-14T11:10:13.783411+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:10:13.783411+0000","last_undegraded":"2023-07-14T11:10:13.783411+0000","last_fullsized":"2023-07-14T11:10:13.783411+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:08:54.674537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736919+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:10:07.736919+0000","last_peered":"2023-07-14T11:10:07.736919+0000","last_clean":"2023-07-14T11:10:07.736919+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:10:07.736919+0000","last_undegraded":"2023-07-14T11:10:07.736919+0000","last_fullsized":"2023-07-14T11:10:07.736919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:34:43.143864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734396+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:10:07.734396+0000","last_peered":"2023-07-14T11:10:07.734396+0000","last_clean":"2023-07-14T11:10:07.734396+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:10:07.734396+0000","last_undegraded":"2023-07-14T11:10:07.734396+0000","last_fullsized":"2023-07-14T11:10:07.734396+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:13:42.740364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":26,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738655+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:10:07.738655+0000","last_peered":"2023-07-14T11:10:07.738655+0000","last_clean":"2023-07-14T11:10:07.738655+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:10:07.738655+0000","last_undegraded":"2023-07-14T11:10:07.738655+0000","last_fullsized":"2023-07-14T11:10:07.738655+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:17:09.808452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737674+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:10:07.737674+0000","last_peered":"2023-07-14T11:10:07.737674+0000","last_clean":"2023-07-14T11:10:07.737674+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:10:07.737674+0000","last_undegraded":"2023-07-14T11:10:07.737674+0000","last_fullsized":"2023-07-14T11:10:07.737674+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:51:08.137181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141293+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:10:08.141293+0000","last_peered":"2023-07-14T11:10:08.141293+0000","last_clean":"2023-07-14T11:10:08.141293+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:10:08.141293+0000","last_undegraded":"2023-07-14T11:10:08.141293+0000","last_fullsized":"2023-07-14T11:10:08.141293+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:33:53.668586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"83'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740367+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:10:07.740367+0000","last_peered":"2023-07-14T11:10:07.740367+0000","last_clean":"2023-07-14T11:10:07.740367+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:10:07.740367+0000","last_undegraded":"2023-07-14T11:10:07.740367+0000","last_fullsized":"2023-07-14T11:10:07.740367+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:41:36.321965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736750+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:10:07.736750+0000","last_peered":"2023-07-14T11:10:07.736750+0000","last_clean":"2023-07-14T11:10:07.736750+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:10:07.736750+0000","last_undegraded":"2023-07-14T11:10:07.736750+0000","last_fullsized":"2023-07-14T11:10:07.736750+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:11:06.323972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739174+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:10:07.739174+0000","last_peered":"2023-07-14T11:10:07.739174+0000","last_clean":"2023-07-14T11:10:07.739174+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:10:07.739174+0000","last_undegraded":"2023-07-14T11:10:07.739174+0000","last_fullsized":"2023-07-14T11:10:07.739174+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:16.916714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"83'2","reported_seq":38,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.827927+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:10:07.827927+0000","last_peered":"2023-07-14T11:10:07.827927+0000","last_clean":"2023-07-14T11:10:07.827927+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:10:07.827927+0000","last_undegraded":"2023-07-14T11:10:07.827927+0000","last_fullsized":"2023-07-14T11:10:07.827927+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:33.874908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737919+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:10:07.737919+0000","last_peered":"2023-07-14T11:10:07.737919+0000","last_clean":"2023-07-14T11:10:07.737919+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:10:07.737919+0000","last_undegraded":"2023-07-14T11:10:07.737919+0000","last_fullsized":"2023-07-14T11:10:07.737919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:56.102535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737624+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:10:07.737624+0000","last_peered":"2023-07-14T11:10:07.737624+0000","last_clean":"2023-07-14T11:10:07.737624+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:10:07.737624+0000","last_undegraded":"2023-07-14T11:10:07.737624+0000","last_fullsized":"2023-07-14T11:10:07.737624+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:49.125150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738992+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:10:07.738992+0000","last_peered":"2023-07-14T11:10:07.738992+0000","last_clean":"2023-07-14T11:10:07.738992+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:10:07.738992+0000","last_undegraded":"2023-07-14T11:10:07.738992+0000","last_fullsized":"2023-07-14T11:10:07.738992+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:49:40.242321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738280+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:10:07.738280+0000","last_peered":"2023-07-14T11:10:07.738280+0000","last_clean":"2023-07-14T11:10:07.738280+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:10:07.738280+0000","last_undegraded":"2023-07-14T11:10:07.738280+0000","last_fullsized":"2023-07-14T11:10:07.738280+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:00:40.108545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141672+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:10:08.141672+0000","last_peered":"2023-07-14T11:10:08.141672+0000","last_clean":"2023-07-14T11:10:08.141672+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:10:08.141672+0000","last_undegraded":"2023-07-14T11:10:08.141672+0000","last_fullsized":"2023-07-14T11:10:08.141672+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:35:25.035847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143146+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:10:08.143146+0000","last_peered":"2023-07-14T11:10:08.143146+0000","last_clean":"2023-07-14T11:10:08.143146+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:10:08.143146+0000","last_undegraded":"2023-07-14T11:10:08.143146+0000","last_fullsized":"2023-07-14T11:10:08.143146+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:00:48.749245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744121+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:10:07.744121+0000","last_peered":"2023-07-14T11:10:07.744121+0000","last_clean":"2023-07-14T11:10:07.744121+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:10:07.744121+0000","last_undegraded":"2023-07-14T11:10:07.744121+0000","last_fullsized":"2023-07-14T11:10:07.744121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:56:30.382410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:27.366892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823702+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:10:07.823702+0000","last_peered":"2023-07-14T11:10:07.823702+0000","last_clean":"2023-07-14T11:10:07.823702+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:10:07.823702+0000","last_undegraded":"2023-07-14T11:10:07.823702+0000","last_fullsized":"2023-07-14T11:10:07.823702+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:42.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739599+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:10:07.739599+0000","last_peered":"2023-07-14T11:10:07.739599+0000","last_clean":"2023-07-14T11:10:07.739599+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:10:07.739599+0000","last_undegraded":"2023-07-14T11:10:07.739599+0000","last_fullsized":"2023-07-14T11:10:07.739599+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:09:31.536272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739315+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:10:07.739315+0000","last_peered":"2023-07-14T11:10:07.739315+0000","last_clean":"2023-07-14T11:10:07.739315+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:10:07.739315+0000","last_undegraded":"2023-07-14T11:10:07.739315+0000","last_fullsized":"2023-07-14T11:10:07.739315+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:06:45.531716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734952+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:10:07.734952+0000","last_peered":"2023-07-14T11:10:07.734952+0000","last_clean":"2023-07-14T11:10:07.734952+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:10:07.734952+0000","last_undegraded":"2023-07-14T11:10:07.734952+0000","last_fullsized":"2023-07-14T11:10:07.734952+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:01.416822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734822+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:10:07.734822+0000","last_peered":"2023-07-14T11:10:07.734822+0000","last_clean":"2023-07-14T11:10:07.734822+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:10:07.734822+0000","last_undegraded":"2023-07-14T11:10:07.734822+0000","last_fullsized":"2023-07-14T11:10:07.734822+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:02:57.300787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737927+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:10:07.737927+0000","last_peered":"2023-07-14T11:10:07.737927+0000","last_clean":"2023-07-14T11:10:07.737927+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:10:07.737927+0000","last_undegraded":"2023-07-14T11:10:07.737927+0000","last_fullsized":"2023-07-14T11:10:07.737927+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:53:58.972191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6118,"num_objects":55,"num_object_clones":0,"num_object_copies":165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55,"num_whiteouts":0,"num_read":11,"num_read_kb":13,"num_write":120,"num_write_kb":41,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":37,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":196608,"data_stored":20082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102,"ondisk_log_size":102,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416403,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.65700000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.62}]},{"osd":2,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.5},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.85099999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.70199999999999996}]},{"osd":7,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.82499999999999996}]},{"osd":8,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76600000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.68400000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.75}]}]},{"osd":10,"up_from":72,"seq":309237645333,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28544,"kb_used_data":1592,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716608,"statfs":{"total":95995035648,"available":95965806592,"internally_reserved":0,"allocated":1630208,"data_stored":1083855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.36099999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.499},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":4.8230000000000004,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.48499999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.38300000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":0.66400000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":1.2749999999999999,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":4.4900000000000002,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.51500000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.624},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.6140000000000001,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.64300000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.68300000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":4.8609999999999998,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":0.55000000000000004}]}]},{"osd":9,"up_from":67,"seq":287762808857,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":1003520,"data_stored":489319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.94099999999999995}]},{"osd":1,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.095,"5min":1.095,"15min":1.095},"last":0.62},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0609999999999999,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.437}]},{"osd":2,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.1910000000000001,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.498},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.59799999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.292,"5min":1.292,"15min":1.292},"last":0.76600000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.70299999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.88,"15min":0.88},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.242,"5min":1.242,"15min":1.242},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":0.57499999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.9119999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.65100000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"min":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.5329999999999999,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.88500000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.91700000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72999999999999998}]}]},{"osd":8,"up_from":59,"seq":253403070495,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27948,"kb_used_data":996,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717204,"statfs":{"total":95995035648,"available":95966416896,"internally_reserved":0,"allocated":1019904,"data_stored":489475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.58699999999999997,"15min":0.59499999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.76600000000000001,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.60499999999999998,"15min":0.61799999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.89000000000000001,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.50800000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.53900000000000003,"15min":0.52900000000000003},"min":{"1min":0.377,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.59099999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.64600000000000002,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.45200000000000001,"15min":0.45500000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.621,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.63400000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.65200000000000002,"15min":0.66100000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.079,"5min":1.079,"15min":1.079},"last":0.91800000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60299999999999998,"15min":0.60699999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.83099999999999996,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.626,"15min":0.629},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.85299999999999998,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.98899999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.56000000000000005,"15min":0.56599999999999995},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.70599999999999996,"5min":1.052,"15min":1.052},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.74199999999999999,"5min":1.135,"15min":1.135},"last":0.76200000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.69999999999999996,"15min":0.70599999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90600000000000003,"5min":1.008,"15min":1.008},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.72999999999999998,"15min":0.73999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90100000000000002,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":1.0109999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.71199999999999997},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.94399999999999995,"5min":1.028,"15min":1.028},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.72499999999999998,"15min":0.72799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.873,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.95799999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.749,"15min":0.751},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":1.0449999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":1.085}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.117}]}]},{"osd":7,"up_from":51,"seq":219043332131,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28516,"kb_used_data":1564,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716636,"statfs":{"total":95995035648,"available":95965835264,"internally_reserved":0,"allocated":1601536,"data_stored":1079988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.60199999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46200000000000002,"15min":0.46500000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.62,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.36199999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65300000000000002,"15min":0.65600000000000003},"min":{"1min":0.45800000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.624,"5min":0.622,"15min":0.622},"min":{"1min":0.47599999999999998,"5min":0.435,"15min":0.435},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.629}]},{"osd":3,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.58399999999999996,"15min":0.57699999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59099999999999997,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.56100000000000005}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.76200000000000001,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.53800000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.72399999999999998,"15min":0.72799999999999998},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.70999999999999996,"15min":0.71499999999999997},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.878,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.65500000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.745,"15min":0.73899999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.72399999999999998,"15min":0.71999999999999997},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.042,"5min":1.042,"15min":1.042},"last":0.72399999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.753,"15min":0.75600000000000001},"min":{"1min":0.57499999999999996,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1,"5min":1,"15min":1},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.71299999999999997,"15min":0.71699999999999997},"min":{"1min":0.51900000000000002,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.73699999999999999}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.0649999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.751},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.70099999999999996}]},{"osd":10,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.71399999999999997}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78200000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495656,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27928,"kb_used_data":976,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717224,"statfs":{"total":95995035648,"available":95966437376,"internally_reserved":0,"allocated":999424,"data_stored":488737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48799999999999999,"15min":0.46999999999999997},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.628,"15min":0.60999999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95899999999999996,"5min":1.151,"15min":1.151},"last":0.67500000000000004}]},{"osd":1,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.67900000000000005,"15min":0.67500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90200000000000002,"5min":1.407,"15min":1.407},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.57799999999999996}]},{"osd":2,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74199999999999999,"15min":0.72899999999999998},"min":{"1min":0.60299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.995,"5min":1.087,"15min":1.087},"last":0.998},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.70399999999999996,"15min":0.70299999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":1.0660000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89000000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.747,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.92200000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.76000000000000001,"15min":0.745},"min":{"1min":0.505,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.091,"5min":1.173,"15min":1.173},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0089999999999999,"5min":1.302,"15min":1.302},"last":0.79700000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.72399999999999998,"15min":0.73899999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86599999999999999,"5min":1.004,"15min":1.004},"last":1.022},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.82699999999999996,"15min":0.81899999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1890000000000001,"5min":1.264,"15min":1.264},"last":1.2150000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.82099999999999995,"15min":0.80100000000000005},"min":{"1min":0.64600000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.91500000000000004,"5min":0.83199999999999996,"15min":0.81799999999999995},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":1.157}]},{"osd":8,"last update":"Fri Jul 14 11:09:49 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.95799999999999996,"15min":0.97499999999999998},"min":{"1min":0.68200000000000005,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.3700000000000001,"5min":1.389,"15min":1.389},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.82399999999999995,"5min":0.95099999999999996,"15min":0.97199999999999998},"min":{"1min":0.60899999999999999,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.1579999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.2010000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.5800000000000001,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.2849999999999999},{"interface":"front","average":{"1min":1.05,"5min":1.05,"15min":1.05},"min":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"max":{"1min":1.466,"5min":1.466,"15min":1.466},"last":1.107}]},{"osd":10,"last update":"Fri Jul 14 11:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"min":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"max":{"1min":1.6319999999999999,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.236},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":1.0900000000000001}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.26}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":492531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.52100000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.90200000000000002,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.54100000000000004,"15min":0.51100000000000001},"min":{"1min":0.35799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.50800000000000001}]},{"osd":2,"last update":"Fri Jul 14 11:09:33 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":0.58299999999999996,"15min":0.54300000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54300000000000004,"15min":0.51100000000000001},"min":{"1min":0.38900000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.72899999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:09:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.56999999999999995,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77700000000000002,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.64600000000000002,"15min":0.58499999999999996},"min":{"1min":0.57099999999999995,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3819999999999997,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.65800000000000003}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.66200000000000003,"15min":0.64200000000000002},"min":{"1min":0.52200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.60899999999999999,"15min":0.59499999999999997},"min":{"1min":0.441,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.0129999999999999,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.69999999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68000000000000005,"15min":0.66300000000000003},"min":{"1min":0.53600000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86799999999999999,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.72999999999999998},"min":{"1min":0.49399999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1180000000000001,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.76700000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.67100000000000004,"15min":0.66000000000000003},"min":{"1min":0.496,"5min":0.316,"15min":0.316},"max":{"1min":0.94999999999999996,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.35199999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.77600000000000002,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59699999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.66000000000000003,"15min":0.64300000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.745},{"interface":"front","average":{"1min":0.871,"5min":0.73899999999999999,"15min":0.71699999999999997},"min":{"1min":0.51600000000000001,"5min":0.441,"15min":0.441},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.79700000000000004}]},{"osd":9,"last update":"Fri Jul 14 11:09:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.67600000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.63300000000000001}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82499999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27936,"kb_used_data":984,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717216,"statfs":{"total":95995035648,"available":95966429184,"internally_reserved":0,"allocated":1007616,"data_stored":489158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58999999999999997,"15min":0.55400000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.94699999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.60399999999999998,"15min":0.55800000000000005},"min":{"1min":0.38100000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":2.1349999999999998,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.51000000000000001}]},{"osd":2,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.56499999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.86099999999999999,"5min":2.528,"15min":2.528},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.61499999999999999,"15min":0.58799999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.0840000000000001,"5min":1.8600000000000001,"15min":1.8600000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Jul 14 11:07:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.436,"15min":0.42399999999999999},"min":{"1min":0.36699999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.46500000000000002,"15min":0.45700000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.42999999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:09:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.64500000000000002,"15min":0.61499999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.0329999999999999,"5min":2.48,"15min":2.48},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.75,"5min":0.70999999999999996,"15min":0.68300000000000005},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.86199999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:09:42 2023","interfaces":[{"interface":"back","average":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.67400000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.68000000000000005,"15min":0.64400000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.028,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":0.84399999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.70599999999999996,"15min":0.69599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99199999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.62},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.71199999999999997,"15min":0.70799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.034,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.66500000000000004}]},{"osd":7,"last update":"Fri Jul 14 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.84199999999999997,"15min":0.84299999999999997},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.1479999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.79100000000000004,"15min":0.78800000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.2290000000000001,"5min":1.2290000000000001,"15min":1.2290000000000001},"last":0.77700000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:09:55 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.85099999999999998,"15min":0.86199999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.1799999999999999,"5min":1.2549999999999999,"15min":1.2549999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.755,"5min":0.86799999999999999,"15min":0.88700000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.1020000000000001,"5min":1.915,"15min":1.915},"last":0.82899999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:09:17 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":0.88500000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.93100000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.873,"15min":0.873},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":1.181,"5min":1.181,"15min":1.181},"last":0.874}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":492292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.49299999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.78300000000000003,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.39500000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81299999999999994,"5min":0.879,"15min":0.879},"last":0.68500000000000005}]},{"osd":1,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.54100000000000004,"15min":0.53100000000000003},"min":{"1min":0.41499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77400000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.56200000000000006,"15min":0.54700000000000004},"min":{"1min":0.374,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.58199999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83099999999999996,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.46899999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.72899999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:21 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58599999999999997,"15min":0.56899999999999995},"min":{"1min":0.48199999999999998,"5min":0.377,"15min":0.377},"max":{"1min":0.86799999999999999,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.622,"15min":0.60599999999999998},"min":{"1min":0.45900000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.71299999999999997}]},{"osd":5,"last update":"Fri Jul 14 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.46500000000000002,"15min":0.45600000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.47899999999999998,"15min":0.46899999999999997},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.60899999999999999},"min":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.66200000000000003,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.81100000000000005}]},{"osd":7,"last update":"Fri Jul 14 11:09:27 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.67200000000000004,"15min":0.66200000000000003},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.69999999999999996,"15min":0.69499999999999995},"min":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.92200000000000004,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.746}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.57099999999999995,"15min":0.55300000000000005},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.623},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.58699999999999997,"15min":0.57999999999999996},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.82699999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.64200000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.84099999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.85799999999999998}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.87}]}]},{"osd":3,"up_from":25,"seq":107374182453,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27932,"kb_used_data":980,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717220,"statfs":{"total":95995035648,"available":95966433280,"internally_reserved":0,"allocated":1003520,"data_stored":488568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45200000000000001,"15min":0.40999999999999998},"min":{"1min":0.33800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.66200000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47599999999999998,"15min":0.436},"min":{"1min":0.36099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81299999999999994,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.44900000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.38100000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.65800000000000003,"15min":0.59699999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89700000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.85999999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65800000000000003,"15min":0.63300000000000001},"min":{"1min":0.504,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94099999999999995,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.67400000000000004,"15min":0.64600000000000002},"min":{"1min":0.504,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.046,"5min":1.069,"15min":1.069},"last":0.57099999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:09:24 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.65100000000000002,"15min":0.627},"min":{"1min":0.50800000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.995,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.90600000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.71099999999999997,"15min":0.68000000000000005},"min":{"1min":0.56000000000000005,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.93899999999999995}]},{"osd":5,"last update":"Fri Jul 14 11:09:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.751,"15min":0.73699999999999999},"min":{"1min":0.57399999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.024,"5min":1.262,"15min":1.262},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.753,"5min":0.75700000000000001,"15min":0.751},"min":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"max":{"1min":1.151,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.52800000000000002}]},{"osd":6,"last update":"Fri Jul 14 11:09:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.745,"15min":0.72799999999999998},"min":{"1min":0.505,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.755,"5min":0.76200000000000001,"15min":0.76300000000000001},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.115,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Fri Jul 14 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.82299999999999995,"15min":0.82899999999999996},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0129999999999999,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.81399999999999995,"15min":0.81000000000000005},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.1040000000000001,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.75700000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:09:15 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.002,"5min":1.002,"15min":1.002},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":1.006}]},{"osd":10,"last update":"Fri Jul 14 11:09:37 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.036,"15min":1.036},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":3.9199999999999999,"5min":3.9199999999999999,"15min":3.9199999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.046}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.92200000000000004}]}]},{"osd":4,"up_from":33,"seq":141733920819,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.373,"15min":0.36199999999999999},"min":{"1min":0.29299999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.40100000000000002,"15min":0.38500000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.35599999999999998}]},{"osd":2,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80500000000000005,"5min":0.877,"15min":0.877},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.59299999999999997,"15min":0.57299999999999995},"min":{"1min":0.42899999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89300000000000002,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.54300000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.59899999999999998,"15min":0.57999999999999996},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.59399999999999997,"15min":0.58299999999999996},"min":{"1min":0.41099999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.77700000000000002}]},{"osd":5,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.59999999999999998,"15min":0.58199999999999996},"min":{"1min":0.45600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85999999999999999,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.65200000000000002,"15min":0.63200000000000001},"min":{"1min":0.505,"5min":0.373,"15min":0.373},"max":{"1min":0.874,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.59399999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.65100000000000002,"15min":0.63500000000000001},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83499999999999996,"5min":1.105,"15min":1.105},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.753,"5min":0.71299999999999997,"15min":0.68100000000000005},"min":{"1min":0.51000000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.90400000000000003,"5min":1.137,"15min":1.137},"last":0.85299999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:09:29 2023","interfaces":[{"interface":"back","average":{"1min":0.752,"5min":0.63100000000000001,"15min":0.61099999999999999},"min":{"1min":0.51500000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.499},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.60399999999999998,"15min":0.58999999999999997},"min":{"1min":0.44800000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.63100000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.52600000000000002,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.90900000000000003,"5min":1.0009999999999999,"15min":1.0009999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.80400000000000005,"15min":0.79900000000000004},"min":{"1min":0.60299999999999998,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0529999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.75700000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:09:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.1240000000000001,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":1},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.103,"5min":1.103,"15min":1.103},"last":0.73799999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:09:34 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.91600000000000004}]},{"osd":11,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.95199999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757293,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28516,"kb_used_data":1564,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716636,"statfs":{"total":95995035648,"available":95965835264,"internally_reserved":0,"allocated":1601536,"data_stored":1079014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.35099999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.33600000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.73499999999999999,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.49099999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57599999999999996,"15min":0.57999999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.68799999999999994,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59399999999999997,"15min":0.58799999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.77400000000000002,"5min":1.466,"15min":1.466},"last":0.62}]},{"osd":2,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.43099999999999999,"15min":0.42599999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.439,"15min":0.434},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.437}]},{"osd":3,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82499999999999996,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.61299999999999999,"15min":0.61499999999999999},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77100000000000002,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.70399999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:09:46 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.69799999999999995,"15min":0.65400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":4.9340000000000002,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.67800000000000005,"15min":0.64000000000000001},"min":{"1min":0.40999999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88400000000000001,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68300000000000005,"15min":0.68899999999999995},"min":{"1min":0.51500000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.75}]},{"osd":7,"last update":"Fri Jul 14 11:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.70399999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.90400000000000003,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":0.73199999999999998,"15min":0.73899999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.90800000000000003,"5min":1.528,"15min":1.528},"last":0.73699999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:09:51 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.629,"15min":0.629},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86099999999999999,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.629,"15min":0.63200000000000001},"min":{"1min":0.371,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.83499999999999996,"5min":0.878,"15min":0.878},"last":0.72099999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.755,"5min":0.755,"15min":0.755},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.82499999999999996}]},{"osd":10,"last update":"Fri Jul 14 11:09:36 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":4.9900000000000002,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.78400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":3828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:10:17.297 INFO:tasks.cephfs.filesystem.ceph_manager:clean! 2023-07-14T11:10:17.298 INFO:tasks.cephadm:Setting up client nodes... 2023-07-14T11:10:17.298 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-07-14T11:10:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:17 smithi130 ceph-mon[129405]: pgmap v295: 129 pgs: 129 active+clean; 583 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s rd, 2.3 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:17 smithi130 ceph-mon[129405]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:active 2023-07-14T11:10:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 2 up:standby-replay 5 up:standby 5 up:active 2023-07-14T11:10:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:10:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:17 smithi040 ceph-mon[97954]: pgmap v295: 129 pgs: 129 active+clean; 583 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s rd, 2.3 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:17 smithi040 ceph-mon[97954]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:active 2023-07-14T11:10:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 2 up:standby-replay 5 up:standby 5 up:active 2023-07-14T11:10:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:10:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:17 smithi045 ceph-mon[128791]: pgmap v295: 129 pgs: 129 active+clean; 583 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.3 KiB/s rd, 2.3 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:17 smithi045 ceph-mon[128791]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] up:active 2023-07-14T11:10:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 2 up:standby-replay 5 up:standby 5 up:active 2023-07-14T11:10:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:10:18.064 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:19.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:18 smithi130 ceph-mon[129405]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:18 smithi040 ceph-mon[97954]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:19.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:18 smithi045 ceph-mon[128791]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:10:19.064 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:10:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:10:18.957+0000 7f3cfd495700 -1 mds.pinger is_rank_lagging: rank=1 was never sent ping request. 2023-07-14T11:10:19.064 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:10:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:10:18.957+0000 7f3cfd495700 -1 mds.pinger is_rank_lagging: rank=2 was never sent ping request. 2023-07-14T11:10:19.064 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:10:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:10:18.957+0000 7f3cfd495700 -1 mds.pinger is_rank_lagging: rank=3 was never sent ping request. 2023-07-14T11:10:19.064 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:10:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:10:18.957+0000 7f3cfd495700 -1 mds.pinger is_rank_lagging: rank=4 was never sent ping request. 2023-07-14T11:10:19.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.616+0000 7f18f8025700 1 -- 172.21.15.40:0/3671051368 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 msgr2=0x7f18f0101200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:19.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.616+0000 7f18f8025700 1 --2- 172.21.15.40:0/3671051368 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0101200 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f18e0009990 tx=0x7f18e0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 -- 172.21.15.40:0/3671051368 shutdown_connections 2023-07-14T11:10:19.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 --2- 172.21.15.40:0/3671051368 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f0101740 0x7f18f0108000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 --2- 172.21.15.40:0/3671051368 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f00ffbb0 0x7f18f00fffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 --2- 172.21.15.40:0/3671051368 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0101200 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 -- 172.21.15.40:0/3671051368 >> 172.21.15.40:0/3671051368 conn(0x7f18f00fb200 msgr2=0x7f18f00fd630 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:19.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 -- 172.21.15.40:0/3671051368 shutdown_connections 2023-07-14T11:10:19.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.617+0000 7f18f8025700 1 -- 172.21.15.40:0/3671051368 wait complete. 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.618+0000 7f18f8025700 1 Processor -- start 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 -- start start 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f00ffbb0 0x7f18f0197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f0101740 0x7f18f019f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18f019f630 con 0x7f18f0100db0 2023-07-14T11:10:19.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18f019f770 con 0x7f18f00ffbb0 2023-07-14T11:10:19.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f8025700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18f019f900 con 0x7f18f0101740 2023-07-14T11:10:19.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f55c0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f5dc1700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f00ffbb0 0x7f18f0197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f55c0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:43154/0 (socket says 172.21.15.40:43154) 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f55c0700 1 -- 172.21.15.40:0/3628680870 learned_addr learned my addr 172.21.15.40:0/3628680870 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.619+0000 7f18f65c2700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f0101740 0x7f18f019f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 -- 172.21.15.40:0/3628680870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f0101740 msgr2=0x7f18f019f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f0101740 0x7f18f019f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 -- 172.21.15.40:0/3628680870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f00ffbb0 msgr2=0x7f18f0197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f00ffbb0 0x7f18f0197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 -- 172.21.15.40:0/3628680870 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e00095f0 con 0x7f18f0100db0 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18f55c0700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f18e0005910 tx=0x7f18e0005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.620+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18e001d070 con 0x7f18f0100db0 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.621+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18f019fb80 con 0x7f18f0100db0 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.621+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f18e0022470 con 0x7f18f0100db0 2023-07-14T11:10:19.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.621+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18e0017650 con 0x7f18f0100db0 2023-07-14T11:10:19.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.621+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18f019fff0 con 0x7f18f0100db0 2023-07-14T11:10:19.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.622+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18f0192280 con 0x7f18f0100db0 2023-07-14T11:10:19.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.625+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f18e0022870 con 0x7f18f0100db0 2023-07-14T11:10:19.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.626+0000 7f18e6ffd700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 0x7f18dc0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:19.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.626+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f18e00ce000 con 0x7f18f0100db0 2023-07-14T11:10:19.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.626+0000 7f18f5dc1700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 0x7f18dc0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:19.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.627+0000 7f18f5dc1700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 0x7f18dc0a3680 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f18f0100c10 tx=0x7f18ec009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:19.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.630+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f18e0097560 con 0x7f18f0100db0 2023-07-14T11:10:19.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.920+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) v1 -- 0x7f18f002c850 con 0x7f18f0100db0 2023-07-14T11:10:19.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.925+0000 7f18e6ffd700 1 -- 172.21.15.40:0/3628680870 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v32) v1 ==== 170+0+59 (secure 0 0 0) 0x7f18e009ab80 con 0x7f18f0100db0 2023-07-14T11:10:19.926 INFO:teuthology.orchestra.run.smithi040.stdout:[client.0] 2023-07-14T11:10:19.927 INFO:teuthology.orchestra.run.smithi040.stdout: key = AQAbLbFkDqvpNhAAYap7GzVoOqit0nO18uVCsQ== 2023-07-14T11:10:19.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 msgr2=0x7f18dc0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:19.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 0x7f18dc0a3680 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f18f0100c10 tx=0x7f18ec009260 comp rx=0 tx=0).stop 2023-07-14T11:10:19.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 msgr2=0x7f18f0197fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f18e0005910 tx=0x7f18e0005740 comp rx=0 tx=0).stop 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 shutdown_connections 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f18dc0a11d0 0x7f18dc0a3680 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f18f0101740 0x7f18f019f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f18f00ffbb0 0x7f18f0197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 --2- 172.21.15.40:0/3628680870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f0100db0 0x7f18f0197fe0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:19.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 >> 172.21.15.40:0/3628680870 conn(0x7f18f00fb200 msgr2=0x7f18f01066a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:19.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 shutdown_connections 2023-07-14T11:10:19.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:19.927+0000 7f18f8025700 1 -- 172.21.15.40:0/3628680870 wait complete. 2023-07-14T11:10:20.037 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:19 smithi040 ceph-mon[97954]: pgmap v296: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:19 smithi130 ceph-mon[129405]: pgmap v296: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:19 smithi045 ceph-mon[128791]: pgmap v296: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 1.7 KiB/s rd, 3.6 KiB/s wr, 13 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:20.363 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:10:20.364 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-07-14T11:10:20.364 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-07-14T11:10:20.468 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-07-14T11:10:20.468 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-07-14T11:10:20.469 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph mgr dump --format=json 2023-07-14T11:10:21.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:20 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-07-14T11:10:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:20 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-07-14T11:10:21.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:20 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-07-14T11:10:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:20 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-07-14T11:10:21.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:20 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-07-14T11:10:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:20 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3628680870' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-07-14T11:10:21.238 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:21 smithi040 ceph-mon[97954]: pgmap v297: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 4.1 KiB/s wr, 16 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:22.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:21 smithi130 ceph-mon[129405]: pgmap v297: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 4.1 KiB/s wr, 16 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:22.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:21 smithi045 ceph-mon[128791]: pgmap v297: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 3.0 KiB/s rd, 4.1 KiB/s wr, 16 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:22.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 -- 172.21.15.40:0/2746367073 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d00ffe00 msgr2=0x7fb1d01001d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:22.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 --2- 172.21.15.40:0/2746367073 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d00ffe00 0x7fb1d01001d0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7fb1cc0099c0 tx=0x7fb1cc009cd0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 -- 172.21.15.40:0/2746367073 shutdown_connections 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 --2- 172.21.15.40:0/2746367073 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d0104ea0 0x7fb1d010ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 --2- 172.21.15.40:0/2746367073 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d00ffe00 0x7fb1d01001d0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 --2- 172.21.15.40:0/2746367073 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d01007a0 0x7fb1d0104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 -- 172.21.15.40:0/2746367073 >> 172.21.15.40:0/2746367073 conn(0x7fb1d00fb660 msgr2=0x7fb1d00fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 -- 172.21.15.40:0/2746367073 shutdown_connections 2023-07-14T11:10:22.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.641+0000 7fb1d869b700 1 -- 172.21.15.40:0/2746367073 wait complete. 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.642+0000 7fb1d869b700 1 Processor -- start 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.642+0000 7fb1d869b700 1 -- start start 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d01007a0 0x7fb1d01983a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 0x7fb1d019c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1d019ce20 con 0x7fb1d0104ea0 2023-07-14T11:10:22.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1d019cf90 con 0x7fb1d00ffe00 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d869b700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1d019d150 con 0x7fb1d01007a0 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6c38700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 0x7fb1d019c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38616/0 (socket says 172.21.15.40:38616) 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 -- 172.21.15.40:0/3850823394 learned_addr learned my addr 172.21.15.40:0/3850823394 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d5c36700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d01007a0 0x7fb1d01983a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:22.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 -- 172.21.15.40:0/3850823394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d01007a0 msgr2=0x7fb1d01983a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d01007a0 0x7fb1d01983a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 -- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 msgr2=0x7fb1d019c810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 0x7fb1d019c810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6437700 1 -- 172.21.15.40:0/3850823394 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb1cc0095f0 con 0x7fb1d00ffe00 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.643+0000 7fb1d6c38700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 0x7fb1d019c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1d6437700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb1cc000c00 tx=0x7fb1cc00bfd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1cc01d070 con 0x7fb1d00ffe00 2023-07-14T11:10:22.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb1d019d3d0 con 0x7fb1d00ffe00 2023-07-14T11:10:22.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb1cc005050 con 0x7fb1d00ffe00 2023-07-14T11:10:22.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1cc004ab0 con 0x7fb1d00ffe00 2023-07-14T11:10:22.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.644+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb1d01a4c90 con 0x7fb1d00ffe00 2023-07-14T11:10:22.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.645+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb1d004e210 con 0x7fb1d00ffe00 2023-07-14T11:10:22.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.647+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb1cc0051c0 con 0x7fb1d00ffe00 2023-07-14T11:10:22.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.649+0000 7fb1c77fe700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 0x7fb1bc0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:22.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.649+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fb1cc0cdd70 con 0x7fb1d00ffe00 2023-07-14T11:10:22.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.649+0000 7fb1d5c36700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 0x7fb1bc0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:22.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.650+0000 7fb1d5c36700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 0x7fb1bc0a3600 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb1d0199480 tx=0x7fb1c0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:22.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.651+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb1cc0971e0 con 0x7fb1d00ffe00 2023-07-14T11:10:22.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.916+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7fb1d01a4e20 con 0x7fb1d00ffe00 2023-07-14T11:10:22.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.922+0000 7fb1c77fe700 1 -- 172.21.15.40:0/3850823394 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v16) v1 ==== 74+0+260036 (secure 0 0 0) 0x7fb1cc09a800 con 0x7fb1d00ffe00 2023-07-14T11:10:22.927 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.926+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 msgr2=0x7fb1bc0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.926+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 0x7fb1bc0a3600 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7fb1d0199480 tx=0x7fb1c0009260 comp rx=0 tx=0).stop 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.926+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 msgr2=0x7fb1d0197e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.926+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb1cc000c00 tx=0x7fb1cc00bfd0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 shutdown_connections 2023-07-14T11:10:22.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb1bc0a1150 0x7fb1bc0a3600 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb1d0104ea0 0x7fb1d019c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb1d00ffe00 0x7fb1d0197e60 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 --2- 172.21.15.40:0/3850823394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb1d01007a0 0x7fb1d01983a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 >> 172.21.15.40:0/3850823394 conn(0x7fb1d00fb660 msgr2=0x7fb1d00fc4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 shutdown_connections 2023-07-14T11:10:22.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:22.927+0000 7fb1d869b700 1 -- 172.21.15.40:0/3850823394 wait complete. 2023-07-14T11:10:23.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:22 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3850823394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-07-14T11:10:23.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:22 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3850823394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-07-14T11:10:23.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:22 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3850823394' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-07-14T11:10:23.434 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":16,"active_gid":14148,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6800","nonce":2087222828},{"type":"v1","addr":"172.21.15.40:6801","nonce":2087222828}]},"active_addr":"172.21.15.40:6801/2087222828","active_change":"2023-07-14T11:03:06.541556+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24109,"name":"z","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]},{"gid":24113,"name":"y","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_password":{"name":"alertmanager_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web password","long_desc":"","tags":[],"see_also":[]},"alertmanager_web_user":{"name":"alertmanager_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Alertmanager web user","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"prometheus_web_password":{"name":"prometheus_web_password","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web password","long_desc":"","tags":[],"see_also":[]},"prometheus_web_user":{"name":"prometheus_web_user","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"Prometheus web user","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.40:8443/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":3,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.40:0","nonce":3852248366}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.40:0","nonce":3580297519}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.40:0","nonce":234853232}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.40:0","nonce":583014137}]}]} 2023-07-14T11:10:23.441 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-07-14T11:10:23.441 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-07-14T11:10:23.442 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd dump --format=json 2023-07-14T11:10:24.179 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:24.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:23 smithi130 ceph-mon[129405]: pgmap v298: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 4.0 KiB/s wr, 20 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:24.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:23 smithi040 ceph-mon[97954]: pgmap v298: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 4.0 KiB/s wr, 20 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:24.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:23 smithi045 ceph-mon[128791]: pgmap v298: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 4.0 KiB/s wr, 20 op/s; 0 B/s, 0 objects/s recovering 2023-07-14T11:10:25.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2342897396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 msgr2=0x7f22d010bea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:25.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2342897396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d010bea0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f22c4009990 tx=0x7f22c4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2342897396 shutdown_connections 2023-07-14T11:10:25.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2342897396 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d010c3e0 0x7f22d010e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2342897396 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d00728b0 0x7f22d0072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2342897396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d010bea0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2342897396 >> 172.21.15.40:0/2342897396 conn(0x7f22d00fb660 msgr2=0x7f22d00fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:25.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.588+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2342897396 shutdown_connections 2023-07-14T11:10:25.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.589+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2342897396 wait complete. 2023-07-14T11:10:25.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 Processor -- start 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 -- start start 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d00728b0 0x7f22d0197df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d010c3e0 0x7f22d019c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22d019ce40 con 0x7f22d00731c0 2023-07-14T11:10:25.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22d019cfb0 con 0x7f22d010c3e0 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.590+0000 7f22d4b7a700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f22d019d170 con 0x7f22d00728b0 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37580/0 (socket says 172.21.15.40:37580) 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 -- 172.21.15.40:0/2568411582 learned_addr learned my addr 172.21.15.40:0/2568411582 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22ced9d700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d010c3e0 0x7f22d019c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22ce59c700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d00728b0 0x7f22d0197df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:25.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 -- 172.21.15.40:0/2568411582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d00728b0 msgr2=0x7f22d0197df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d00728b0 0x7f22d0197df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 -- 172.21.15.40:0/2568411582 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d010c3e0 msgr2=0x7f22d019c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d010c3e0 0x7f22d019c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 -- 172.21.15.40:0/2568411582 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22c40095f0 con 0x7f22d00731c0 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.591+0000 7f22cdd9b700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f22c40095c0 tx=0x7f22c400f690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.592+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22c401d070 con 0x7f22d00731c0 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.592+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f22d019d3f0 con 0x7f22d00731c0 2023-07-14T11:10:25.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.592+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f22c4005050 con 0x7f22d00731c0 2023-07-14T11:10:25.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.592+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22c4004a90 con 0x7f22d00731c0 2023-07-14T11:10:25.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.592+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22d01a4c20 con 0x7f22d00731c0 2023-07-14T11:10:25.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.593+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f22d004e210 con 0x7f22d00731c0 2023-07-14T11:10:25.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.595+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f22c4005430 con 0x7f22d00731c0 2023-07-14T11:10:25.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.597+0000 7f22bf7fe700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 0x7f22b80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:25.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.597+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f22c40cdf10 con 0x7f22d00731c0 2023-07-14T11:10:25.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.597+0000 7f22ce59c700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 0x7f22b80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:25.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.598+0000 7f22ce59c700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 0x7f22b80a3600 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f22c0005950 tx=0x7f22c000b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:25.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.599+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f22c4097380 con 0x7f22d00731c0 2023-07-14T11:10:25.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:25 smithi130 ceph-mon[129405]: pgmap v299: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2023-07-14T11:10:25.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:25 smithi040 ceph-mon[97954]: pgmap v299: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2023-07-14T11:10:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.807+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f22d01a4db0 con 0x7f22d00731c0 2023-07-14T11:10:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.809+0000 7f22bf7fe700 1 -- 172.21.15.40:0/2568411582 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+24369 (secure 0 0 0) 0x7f22c409a9a0 con 0x7f22d00731c0 2023-07-14T11:10:25.811 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:25.811 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":90,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:10:07.729799+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 msgr2=0x7f22b80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 0x7f22b80a3600 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f22c0005950 tx=0x7f22c000b410 comp rx=0 tx=0).stop 2023-07-14T11:10:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 msgr2=0x7f22d0198330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f22c40095c0 tx=0x7f22c400f690 comp rx=0 tx=0).stop 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 shutdown_connections 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f22b80a1150 0x7f22b80a3600 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22d010c3e0 0x7f22d019c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22d00728b0 0x7f22d0197df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 --2- 172.21.15.40:0/2568411582 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22d00731c0 0x7f22d0198330 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.811+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 >> 172.21.15.40:0/2568411582 conn(0x7f22d00fb660 msgr2=0x7f22d010e480 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.812+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 shutdown_connections 2023-07-14T11:10:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:25.812+0000 7f22d4b7a700 1 -- 172.21.15.40:0/2568411582 wait complete. 2023-07-14T11:10:25.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:25 smithi045 ceph-mon[128791]: pgmap v299: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 4.0 KiB/s wr, 22 op/s 2023-07-14T11:10:26.273 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-07-14T11:10:26.273 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd dump --format=json 2023-07-14T11:10:26.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:26 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2568411582' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:26.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:26 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2568411582' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:26.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:26 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2568411582' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:27.006 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:27.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:27 smithi040 ceph-mon[97954]: pgmap v300: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 3.9 KiB/s wr, 21 op/s 2023-07-14T11:10:27.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:27 smithi130 ceph-mon[129405]: pgmap v300: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 3.9 KiB/s wr, 21 op/s 2023-07-14T11:10:27.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:27 smithi045 ceph-mon[128791]: pgmap v300: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 3.9 KiB/s wr, 21 op/s 2023-07-14T11:10:28.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 -- 172.21.15.40:0/1982702367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d8104ea0 msgr2=0x7f16d810ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 --2- 172.21.15.40:0/1982702367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d8104ea0 0x7f16d810ca20 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f16d000b0d0 tx=0x7f16d000b3e0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 -- 172.21.15.40:0/1982702367 shutdown_connections 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 --2- 172.21.15.40:0/1982702367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d8104ea0 0x7f16d810ca20 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 --2- 172.21.15.40:0/1982702367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d81001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 --2- 172.21.15.40:0/1982702367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d81007a0 0x7f16d8104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.423+0000 7f16df010700 1 -- 172.21.15.40:0/1982702367 >> 172.21.15.40:0/1982702367 conn(0x7f16d80fb660 msgr2=0x7f16d80fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:28.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.424+0000 7f16df010700 1 -- 172.21.15.40:0/1982702367 shutdown_connections 2023-07-14T11:10:28.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.424+0000 7f16df010700 1 -- 172.21.15.40:0/1982702367 wait complete. 2023-07-14T11:10:28.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 Processor -- start 2023-07-14T11:10:28.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 -- start start 2023-07-14T11:10:28.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d8198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16d819cfe0 con 0x7f16d81007a0 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16d819d150 con 0x7f16d8104ea0 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16df010700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16d819d310 con 0x7f16d80ffe00 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16d7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16dd5ad700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:28.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16dd5ad700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:42110/0 (socket says 172.21.15.40:42110) 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16dd5ad700 1 -- 172.21.15.40:0/3220817852 learned_addr learned my addr 172.21.15.40:0/3220817852 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.425+0000 7f16d7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37608/0 (socket says 172.21.15.40:37608) 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dcdac700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d8198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 -- 172.21.15.40:0/3220817852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 msgr2=0x7f16d8198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d8198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 -- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 msgr2=0x7f16d8198560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:28.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 -- 172.21.15.40:0/3220817852 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16d0009d00 con 0x7f16d8104ea0 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dcdac700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d8198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16d7fff700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16dd5ad700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f16d0007820 tx=0x7f16d0007850 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.426+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d000e040 con 0x7f16d8104ea0 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.427+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f16d000bd30 con 0x7f16d8104ea0 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.427+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16d819d590 con 0x7f16d8104ea0 2023-07-14T11:10:28.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.427+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d001d470 con 0x7f16d8104ea0 2023-07-14T11:10:28.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.427+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16d81a4e50 con 0x7f16d8104ea0 2023-07-14T11:10:28.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.428+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16d804e210 con 0x7f16d8104ea0 2023-07-14T11:10:28.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.430+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f16d0004a50 con 0x7f16d8104ea0 2023-07-14T11:10:28.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.431+0000 7f16d5ffb700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 0x7f16c80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:28.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.432+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f16d00ce530 con 0x7f16d8104ea0 2023-07-14T11:10:28.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.432+0000 7f16dcdac700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 0x7f16c80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:28.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.432+0000 7f16dcdac700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 0x7f16c80a3600 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009cb0 tx=0x7f16cc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:28.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.434+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f16d00979a0 con 0x7f16d8104ea0 2023-07-14T11:10:28.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.643+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f16d81a4fe0 con 0x7f16d8104ea0 2023-07-14T11:10:28.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.646+0000 7f16d5ffb700 1 -- 172.21.15.40:0/3220817852 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v90) v1 ==== 74+0+24369 (secure 0 0 0) 0x7f16d009afc0 con 0x7f16d8104ea0 2023-07-14T11:10:28.648 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:10:28.648 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":90,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:10:07.729799+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:10:28.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 msgr2=0x7f16c80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:28.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 0x7f16c80a3600 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009cb0 tx=0x7f16cc009260 comp rx=0 tx=0).stop 2023-07-14T11:10:28.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 msgr2=0x7f16d819c9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f16d0007820 tx=0x7f16d0007850 comp rx=0 tx=0).stop 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 shutdown_connections 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f16c80a1150 0x7f16c80a3600 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16d8104ea0 0x7f16d819c9d0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16d80ffe00 0x7f16d8198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 --2- 172.21.15.40:0/3220817852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16d81007a0 0x7f16d8198560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:28.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.648+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 >> 172.21.15.40:0/3220817852 conn(0x7f16d80fb660 msgr2=0x7f16d810af80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:28.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.649+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 shutdown_connections 2023-07-14T11:10:28.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:28.649+0000 7f16df010700 1 -- 172.21.15.40:0/3220817852 wait complete. 2023-07-14T11:10:29.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:28 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3220817852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:29.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:28 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3220817852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:29.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:28 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3220817852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:10:29.106 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.0 flush_pg_stats 2023-07-14T11:10:29.107 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.1 flush_pg_stats 2023-07-14T11:10:29.107 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.2 flush_pg_stats 2023-07-14T11:10:29.108 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.3 flush_pg_stats 2023-07-14T11:10:29.108 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.4 flush_pg_stats 2023-07-14T11:10:29.109 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.5 flush_pg_stats 2023-07-14T11:10:29.109 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.6 flush_pg_stats 2023-07-14T11:10:29.110 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.7 flush_pg_stats 2023-07-14T11:10:29.111 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.8 flush_pg_stats 2023-07-14T11:10:29.111 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.9 flush_pg_stats 2023-07-14T11:10:29.112 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.10 flush_pg_stats 2023-07-14T11:10:29.112 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph tell osd.11 flush_pg_stats 2023-07-14T11:10:29.728 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:29 smithi040 ceph-mon[97954]: pgmap v301: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 2.2 KiB/s wr, 17 op/s 2023-07-14T11:10:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:29 smithi130 ceph-mon[129405]: pgmap v301: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 2.2 KiB/s wr, 17 op/s 2023-07-14T11:10:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:29 smithi045 ceph-mon[128791]: pgmap v301: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 2.2 KiB/s wr, 17 op/s 2023-07-14T11:10:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.126 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.127 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.155 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.287 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.484 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.486 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.487 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.698 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:31.741 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:31 smithi040 ceph-mon[97954]: pgmap v302: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 853 B/s wr, 14 op/s 2023-07-14T11:10:31.745 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:31 smithi130 ceph-mon[129405]: pgmap v302: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 853 B/s wr, 14 op/s 2023-07-14T11:10:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:31 smithi045 ceph-mon[128791]: pgmap v302: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 6.9 KiB/s rd, 853 B/s wr, 14 op/s 2023-07-14T11:10:32.159 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:32.192 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:34.005 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:33 smithi040 ceph-mon[97954]: pgmap v303: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 0 B/s wr, 11 op/s 2023-07-14T11:10:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:33 smithi130 ceph-mon[129405]: pgmap v303: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 0 B/s wr, 11 op/s 2023-07-14T11:10:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:33 smithi045 ceph-mon[128791]: pgmap v303: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 0 B/s wr, 11 op/s 2023-07-14T11:10:35.511 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:35 smithi040 ceph-mon[97954]: pgmap v304: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:35 smithi130 ceph-mon[129405]: pgmap v304: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:35 smithi045 ceph-mon[128791]: pgmap v304: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:36.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.944+0000 7f995be10700 1 -- 172.21.15.40:0/1400837372 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f99540ff290 msgr2=0x7f995410bff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.944+0000 7f995be10700 1 --2- 172.21.15.40:0/1400837372 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f99540ff290 0x7f995410bff0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f9944009990 tx=0x7f9944009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 -- 172.21.15.40:0/1400837372 shutdown_connections 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 --2- 172.21.15.40:0/1400837372 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f995410c530 0x7f995410e910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 --2- 172.21.15.40:0/1400837372 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540fe980 0x7f99540fed50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 --2- 172.21.15.40:0/1400837372 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f99540ff290 0x7f995410bff0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 -- 172.21.15.40:0/1400837372 >> 172.21.15.40:0/1400837372 conn(0x7f9954075b70 msgr2=0x7f9954075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 -- 172.21.15.40:0/1400837372 shutdown_connections 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.945+0000 7f995be10700 1 -- 172.21.15.40:0/1400837372 wait complete. 2023-07-14T11:10:36.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.946+0000 7f995be10700 1 Processor -- start 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.946+0000 7f995be10700 1 -- start start 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f99540fe980 0x7f9954197ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 0x7f9954198410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f995410c530 0x7f995419c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f995419ce90 con 0x7f99540fe980 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f995419d000 con 0x7f995410c530 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995be10700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f995419d1c0 con 0x7f99540ff290 2023-07-14T11:10:36.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f99593ab700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 0x7f9954198410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f99593ab700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 0x7f9954198410 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33754/0 (socket says 172.21.15.40:33754) 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f99593ab700 1 -- 172.21.15.40:0/2222381451 learned_addr learned my addr 172.21.15.40:0/2222381451 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f9959bac700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f99540fe980 0x7f9954197ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.947+0000 7f995a3ad700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f995410c530 0x7f995419c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 -- 172.21.15.40:0/2222381451 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f995410c530 msgr2=0x7f995419c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f995410c530 0x7f995419c880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 -- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f99540fe980 msgr2=0x7f9954197ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f99540fe980 0x7f9954197ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99440095f0 con 0x7f99540ff290 2023-07-14T11:10:36.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f99593ab700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 0x7f9954198410 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9944000c00 tx=0x7f9944005720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:36.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f994401d070 con 0x7f99540ff290 2023-07-14T11:10:36.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f995419d440 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f995419d700 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9944022470 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.948+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9944017630 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.949+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9938000ff0 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.952+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f99440225e0 con 0x7f99540ff290 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.953+0000 7f994affd700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f99400a11d0 0x7f99400a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:36.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.953+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f99440cea90 con 0x7f99540ff290 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f9959bac700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f99400a11d0 0x7f99400a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f994affd700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] conn(0x7f99400ac0b0 0x7f99400ae540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f99400aec10 con 0x7f99400ac0b0 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f99440cee00 con 0x7f99540ff290 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f995a3ad700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] conn(0x7f99400ac0b0 0x7f99400ae540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.954+0000 7f9959bac700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f99400a11d0 0x7f99400a3680 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f9950009c60 tx=0x7f9950009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.955+0000 7f995a3ad700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] conn(0x7f99400ac0b0 0x7f99400ae540 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:36.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.955+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== osd.3 v2:172.21.15.40:6810/741691351 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f99400aec10 con 0x7f99400ac0b0 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.972+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 --> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9938002d70 con 0x7f99400ac0b0 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f994affd700 1 -- 172.21.15.40:0/2222381451 <== osd.3 v2:172.21.15.40:6810/741691351 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f9938002d70 con 0x7f99400ac0b0 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] conn(0x7f99400ac0b0 msgr2=0x7f99400ae540 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] conn(0x7f99400ac0b0 0x7f99400ae540 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f99400a11d0 msgr2=0x7f99400a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f99400a11d0 0x7f99400a3680 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f9950009c60 tx=0x7f9950009260 comp rx=0 tx=0).stop 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 msgr2=0x7f9954198410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 --2- 172.21.15.40:0/2222381451 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f99540ff290 0x7f9954198410 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f9944000c00 tx=0x7f9944005720 comp rx=0 tx=0).stop 2023-07-14T11:10:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995a3ad700 1 -- 172.21.15.40:0/2222381451 reap_dead start 2023-07-14T11:10:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 shutdown_connections 2023-07-14T11:10:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.973+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 >> 172.21.15.40:0/2222381451 conn(0x7f9954075b70 msgr2=0x7f99540fc3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.974+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 shutdown_connections 2023-07-14T11:10:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:35.974+0000 7f995be10700 1 -- 172.21.15.40:0/2222381451 wait complete. 2023-07-14T11:10:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:37 smithi040 ceph-mon[97954]: pgmap v305: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:38.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:37 smithi130 ceph-mon[129405]: pgmap v305: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:38.064 INFO:teuthology.orchestra.run.smithi040.stdout:107374182458 2023-07-14T11:10:38.064 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.3 2023-07-14T11:10:38.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:37 smithi045 ceph-mon[128791]: pgmap v305: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:38.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.532+0000 7f55bac6a700 1 -- 172.21.15.40:0/3332504124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b41007b0 msgr2=0x7f55b4104780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.532+0000 7f55bac6a700 1 --2- 172.21.15.40:0/3332504124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b41007b0 0x7f55b4104780 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f55a000fb20 tx=0x7f55a000fe30 comp rx=0 tx=0).stop 2023-07-14T11:10:38.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 -- 172.21.15.40:0/3332504124 shutdown_connections 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 --2- 172.21.15.40:0/3332504124 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b4104eb0 0x7f55b410ca30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 --2- 172.21.15.40:0/3332504124 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40ffe10 0x7f55b41001e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 --2- 172.21.15.40:0/3332504124 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b41007b0 0x7f55b4104780 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 -- 172.21.15.40:0/3332504124 >> 172.21.15.40:0/3332504124 conn(0x7f55b40fb690 msgr2=0x7f55b40fdaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 -- 172.21.15.40:0/3332504124 shutdown_connections 2023-07-14T11:10:38.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.533+0000 7f55bac6a700 1 -- 172.21.15.40:0/3332504124 wait complete. 2023-07-14T11:10:38.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.534+0000 7f55bac6a700 1 Processor -- start 2023-07-14T11:10:38.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 -- start start 2023-07-14T11:10:38.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40ffe10 0x7f55b4199820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b41007b0 0x7f55b4199d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 0x7f55b41938f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b4193f60 con 0x7f55b41007b0 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b41940d0 con 0x7f55b40ffe10 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.535+0000 7f55bac6a700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b4194290 con 0x7f55b4104eb0 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 0x7f55b41938f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 0x7f55b41938f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33784/0 (socket says 172.21.15.40:33784) 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 -- 172.21.15.40:0/90322546 learned_addr learned my addr 172.21.15.40:0/90322546 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:38.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b8a06700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40ffe10 0x7f55b4199820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b3fff700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b41007b0 0x7f55b4199d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 -- 172.21.15.40:0/90322546 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40ffe10 msgr2=0x7f55b4199820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40ffe10 0x7f55b4199820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 -- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b41007b0 msgr2=0x7f55b4199d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b41007b0 0x7f55b4199d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.536+0000 7f55b9207700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55a000f750 con 0x7f55b4104eb0 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.537+0000 7f55b9207700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 0x7f55b41938f0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f55ac007ee0 tx=0x7f55ac00d3b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.537+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55ac017070 con 0x7f55b4104eb0 2023-07-14T11:10:38.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.537+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f55ac00f040 con 0x7f55b4104eb0 2023-07-14T11:10:38.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.537+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55ac005740 con 0x7f55b4104eb0 2023-07-14T11:10:38.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.538+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55b4194570 con 0x7f55b4104eb0 2023-07-14T11:10:38.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.538+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55b41a0810 con 0x7f55b4104eb0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.539+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f55b404e210 con 0x7f55b4104eb0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.544+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f55ac00dc60 con 0x7f55b4104eb0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.546+0000 7f55b1ffb700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f55a40a11d0 0x7f55a40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.546+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f55ac0cca30 con 0x7f55b4104eb0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.546+0000 7f55b1ffb700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] conn(0x7f55a40ac1e0 0x7f55a40ae670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.547+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f55a40aed40 con 0x7f55a40ac1e0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.547+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f55ac0cce40 con 0x7f55b4104eb0 2023-07-14T11:10:38.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.547+0000 7f55b8a06700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f55a40a11d0 0x7f55a40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.547+0000 7f55b3fff700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] conn(0x7f55a40ac1e0 0x7f55a40ae670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.548+0000 7f55b3fff700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] conn(0x7f55a40ac1e0 0x7f55a40ae670 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.548+0000 7f55b8a06700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f55a40a11d0 0x7f55a40a3680 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f55a8009c60 tx=0x7f55a8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.549+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== osd.0 v2:172.21.15.40:6802/2649857957 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f55a40aed40 con 0x7f55a40ac1e0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.579+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 --> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f55b40fcb30 con 0x7f55a40ac1e0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.580+0000 7f55b1ffb700 1 -- 172.21.15.40:0/90322546 <== osd.0 v2:172.21.15.40:6802/2649857957 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f55b40fcb30 con 0x7f55a40ac1e0 2023-07-14T11:10:38.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.584+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] conn(0x7f55a40ac1e0 msgr2=0x7f55a40ae670 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.584+0000 7f55bac6a700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] conn(0x7f55a40ac1e0 0x7f55a40ae670 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f55a40a11d0 msgr2=0x7f55a40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f55a40a11d0 0x7f55a40a3680 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f55a8009c60 tx=0x7f55a8009260 comp rx=0 tx=0).stop 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 msgr2=0x7f55b41938f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 --2- 172.21.15.40:0/90322546 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4104eb0 0x7f55b41938f0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f55ac007ee0 tx=0x7f55ac00d3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55b9207700 1 -- 172.21.15.40:0/90322546 reap_dead start 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 shutdown_connections 2023-07-14T11:10:38.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 >> 172.21.15.40:0/90322546 conn(0x7f55b40fb690 msgr2=0x7f55b40fc160 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:38.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 shutdown_connections 2023-07-14T11:10:38.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.585+0000 7f55bac6a700 1 -- 172.21.15.40:0/90322546 wait complete. 2023-07-14T11:10:38.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 -- 172.21.15.40:0/2378554432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 msgr2=0x7f6e9410bff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 --2- 172.21.15.40:0/2378554432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 0x7f6e9410bff0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f6e84009990 tx=0x7f6e84009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 -- 172.21.15.40:0/2378554432 shutdown_connections 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 --2- 172.21.15.40:0/2378554432 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6e9410c530 0x7f6e9410e910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 --2- 172.21.15.40:0/2378554432 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e940fed50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 --2- 172.21.15.40:0/2378554432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 0x7f6e9410bff0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.379+0000 7f6e9c564700 1 -- 172.21.15.40:0/2378554432 >> 172.21.15.40:0/2378554432 conn(0x7f6e94075b70 msgr2=0x7f6e94075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.381+0000 7f6e9c564700 1 -- 172.21.15.40:0/2378554432 shutdown_connections 2023-07-14T11:10:38.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.381+0000 7f6e9c564700 1 -- 172.21.15.40:0/2378554432 wait complete. 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.383+0000 7f6e9c564700 1 Processor -- start 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.383+0000 7f6e9c564700 1 -- start start 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e9410bd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 0x7f6e94102600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6e9410c530 0x7f6e94102b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e941031a0 con 0x7f6e940ff290 2023-07-14T11:10:38.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e94103310 con 0x7f6e940fe980 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9c564700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e941034d0 con 0x7f6e9410c530 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9a300700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e9410bd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9ab01700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6e9410c530 0x7f6e94102b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9a300700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e9410bd10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:47850/0 (socket says 172.21.15.40:47850) 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.384+0000 7f6e9a300700 1 -- 172.21.15.40:0/3089313306 learned_addr learned my addr 172.21.15.40:0/3089313306 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e99aff700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 0x7f6e94102600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 -- 172.21.15.40:0/3089313306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6e9410c530 msgr2=0x7f6e94102b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6e9410c530 0x7f6e94102b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 -- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 msgr2=0x7f6e94102600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e940ff290 0x7f6e94102600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e90009520 con 0x7f6e940fe980 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.385+0000 7f6e9a300700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e9410bd10 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f6e9000eb30 tx=0x7f6e9000ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.386+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e90009880 con 0x7f6e940fe980 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.386+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6e900044e0 con 0x7f6e940fe980 2023-07-14T11:10:38.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.386+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6e90003e90 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.386+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e840095f0 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.387+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e9406a3f0 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.388+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f6e94101eb0 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.392+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6e90004000 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e8b7fe700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6e800a1220 0x7f6e800a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f6e90014070 con 0x7f6e940fe980 2023-07-14T11:10:38.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e8b7fe700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] conn(0x7f6e800ac0c0 0x7f6e800ae550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f6e800aec20 con 0x7f6e800ac0c0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6e900cd4a0 con 0x7f6e940fe980 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.394+0000 7f6e99aff700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6e800a1220 0x7f6e800a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.395+0000 7f6e9ab01700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] conn(0x7f6e800ac0c0 0x7f6e800ae550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.396+0000 7f6e99aff700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6e800a1220 0x7f6e800a36d0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f6e84006010 tx=0x7f6e84005c00 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.396+0000 7f6e9ab01700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] conn(0x7f6e800ac0c0 0x7f6e800ae550 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.397+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== osd.1 v2:172.21.15.45:6800/21717094 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f6e800aec20 con 0x7f6e800ac0c0 2023-07-14T11:10:38.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.432+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 --> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f6e9404e210 con 0x7f6e800ac0c0 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.432+0000 7f6e8b7fe700 1 -- 172.21.15.40:0/3089313306 <== osd.1 v2:172.21.15.45:6800/21717094 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f6e9404e210 con 0x7f6e800ac0c0 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] conn(0x7f6e800ac0c0 msgr2=0x7f6e800ae550 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] conn(0x7f6e800ac0c0 0x7f6e800ae550 crc :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6e800a1220 msgr2=0x7f6e800a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6e800a1220 0x7f6e800a36d0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f6e84006010 tx=0x7f6e84005c00 comp rx=0 tx=0).stop 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 msgr2=0x7f6e9410bd10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 --2- 172.21.15.40:0/3089313306 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6e940fe980 0x7f6e9410bd10 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f6e9000eb30 tx=0x7f6e9000ee40 comp rx=0 tx=0).stop 2023-07-14T11:10:38.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9ab01700 1 -- 172.21.15.40:0/3089313306 reap_dead start 2023-07-14T11:10:38.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 shutdown_connections 2023-07-14T11:10:38.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 >> 172.21.15.40:0/3089313306 conn(0x7f6e94075b70 msgr2=0x7f6e940fc3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:38.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 shutdown_connections 2023-07-14T11:10:38.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.435+0000 7f6e9c564700 1 -- 172.21.15.40:0/3089313306 wait complete. 2023-07-14T11:10:38.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 -- 172.21.15.40:0/4046883921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc100df0 msgr2=0x7f4ecc1011c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4046883921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc100df0 0x7f4ecc1011c0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f4ec80098f0 tx=0x7f4ec8009c00 comp rx=0 tx=0).stop 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 -- 172.21.15.40:0/4046883921 shutdown_connections 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4046883921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc10a140 0x7f4ecc10c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4046883921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc100df0 0x7f4ecc1011c0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4046883921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc101700 0x7f4ecc109c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 -- 172.21.15.40:0/4046883921 >> 172.21.15.40:0/4046883921 conn(0x7f4ecc075b70 msgr2=0x7f4ecc075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:38.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 -- 172.21.15.40:0/4046883921 shutdown_connections 2023-07-14T11:10:38.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.976+0000 7f4ed3547700 1 -- 172.21.15.40:0/4046883921 wait complete. 2023-07-14T11:10:38.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.977+0000 7f4ed3547700 1 Processor -- start 2023-07-14T11:10:38.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 -- start start 2023-07-14T11:10:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 0x7f4ecc19bad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc101700 0x7f4ecc19c010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc10a140 0x7f4ecc195b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ecc1961c0 con 0x7f4ecc10a140 2023-07-14T11:10:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ecc196330 con 0x7f4ecc100df0 2023-07-14T11:10:38.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.978+0000 7f4ed3547700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4ecc1964f0 con 0x7f4ecc101700 2023-07-14T11:10:38.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 0x7f4ecc19bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed0ae2700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc101700 0x7f4ecc19c010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 0x7f4ecc19bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:47904/0 (socket says 172.21.15.40:47904) 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 -- 172.21.15.40:0/4274959420 learned_addr learned my addr 172.21.15.40:0/4274959420 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed1ae4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc10a140 0x7f4ecc195b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 -- 172.21.15.40:0/4274959420 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc101700 msgr2=0x7f4ecc19c010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc101700 0x7f4ecc19c010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 -- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc10a140 msgr2=0x7f4ecc195b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:38.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4ecc10a140 0x7f4ecc195b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:38.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4ec8009520 con 0x7f4ecc100df0 2023-07-14T11:10:38.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed0ae2700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4ecc101700 0x7f4ecc19c010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:38.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.979+0000 7f4ed12e3700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 0x7f4ecc19bad0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f4ec8009810 tx=0x7f4ec800fa00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.980+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ec801c070 con 0x7f4ecc100df0 2023-07-14T11:10:38.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.980+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4ec800bbd0 con 0x7f4ecc100df0 2023-07-14T11:10:38.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.980+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4ecc196770 con 0x7f4ecc100df0 2023-07-14T11:10:38.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.980+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ec8004930 con 0x7f4ecc100df0 2023-07-14T11:10:38.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.980+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4ecc1a2ac0 con 0x7f4ecc100df0 2023-07-14T11:10:38.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.981+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f4ecc10fd70 con 0x7f4ecc100df0 2023-07-14T11:10:38.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.984+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4ec8020070 con 0x7f4ecc100df0 2023-07-14T11:10:38.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.986+0000 7f4ec27fc700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4eb80a11d0 0x7f4eb80a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.986+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f4ec80cdf90 con 0x7f4ecc100df0 2023-07-14T11:10:38.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ed0ae2700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4eb80a11d0 0x7f4eb80a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ec27fc700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] conn(0x7f4eb80ac1e0 0x7f4eb80ae670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:38.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f4eb80aed40 con 0x7f4eb80ac1e0 2023-07-14T11:10:38.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4ec80ce300 con 0x7f4ecc100df0 2023-07-14T11:10:38.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ed1ae4700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] conn(0x7f4eb80ac1e0 0x7f4eb80ae670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:38.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.987+0000 7f4ed0ae2700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4eb80a11d0 0x7f4eb80a3680 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4ecc10c820 tx=0x7f4ebc009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.988+0000 7f4ed1ae4700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] conn(0x7f4eb80ac1e0 0x7f4eb80ae670 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.10 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:38.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.989+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== osd.10 v2:172.21.15.45:6824/3539261425 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f4eb80aed40 con 0x7f4eb80ac1e0 2023-07-14T11:10:39.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.013+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 --> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f4ecc04e210 con 0x7f4eb80ac1e0 2023-07-14T11:10:39.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.013+0000 7f4ec27fc700 1 -- 172.21.15.40:0/4274959420 <== osd.10 v2:172.21.15.45:6824/3539261425 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f4ecc04e210 con 0x7f4eb80ac1e0 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] conn(0x7f4eb80ac1e0 msgr2=0x7f4eb80ae670 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] conn(0x7f4eb80ac1e0 0x7f4eb80ae670 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4eb80a11d0 msgr2=0x7f4eb80a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4eb80a11d0 0x7f4eb80a3680 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4ecc10c820 tx=0x7f4ebc009310 comp rx=0 tx=0).stop 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 msgr2=0x7f4ecc19bad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 --2- 172.21.15.40:0/4274959420 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4ecc100df0 0x7f4ecc19bad0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f4ec8009810 tx=0x7f4ec800fa00 comp rx=0 tx=0).stop 2023-07-14T11:10:39.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed1ae4700 1 -- 172.21.15.40:0/4274959420 reap_dead start 2023-07-14T11:10:39.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 shutdown_connections 2023-07-14T11:10:39.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 >> 172.21.15.40:0/4274959420 conn(0x7f4ecc075b70 msgr2=0x7f4ecc101b70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 shutdown_connections 2023-07-14T11:10:39.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.014+0000 7f4ed3547700 1 -- 172.21.15.40:0/4274959420 wait complete. 2023-07-14T11:10:39.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 -- 172.21.15.40:0/2902150004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc0641007a0 msgr2=0x7fc064104770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 --2- 172.21.15.40:0/2902150004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc0641007a0 0x7fc064104770 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7fc050009990 tx=0x7fc050009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 -- 172.21.15.40:0/2902150004 shutdown_connections 2023-07-14T11:10:39.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 --2- 172.21.15.40:0/2902150004 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc064104ea0 0x7fc06410ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 --2- 172.21.15.40:0/2902150004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0640ffe00 0x7fc0641001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 --2- 172.21.15.40:0/2902150004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc0641007a0 0x7fc064104770 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 -- 172.21.15.40:0/2902150004 >> 172.21.15.40:0/2902150004 conn(0x7fc0640fb660 msgr2=0x7fc0640fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 -- 172.21.15.40:0/2902150004 shutdown_connections 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.066+0000 7fc06b115700 1 -- 172.21.15.40:0/2902150004 wait complete. 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.067+0000 7fc06b115700 1 Processor -- start 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 -- start start 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 0x7fc064072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0641007a0 0x7fc064072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc064104ea0 0x7fc064073290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0641a41c0 con 0x7fc0640ffe00 2023-07-14T11:10:39.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0641a4330 con 0x7fc064104ea0 2023-07-14T11:10:39.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc06b115700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0641a44f0 con 0x7fc0641007a0 2023-07-14T11:10:39.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc068eb1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 0x7fc064072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc068eb1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 0x7fc064072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45426/0 (socket says 172.21.15.40:45426) 2023-07-14T11:10:39.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc068eb1700 1 -- 172.21.15.40:0/2833661206 learned_addr learned my addr 172.21.15.40:0/2833661206 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:39.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.068+0000 7fc063fff700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0641007a0 0x7fc064072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc0696b2700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc064104ea0 0x7fc064073290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 -- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0641007a0 msgr2=0x7fc064072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0641007a0 0x7fc064072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 -- 172.21.15.40:0/2833661206 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc064104ea0 msgr2=0x7fc064073290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc064104ea0 0x7fc064073290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0500095f0 con 0x7fc0640ffe00 2023-07-14T11:10:39.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc063fff700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc0641007a0 0x7fc064072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc068eb1700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 0x7fc064072810 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fc05800e9a0 tx=0x7fc05800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc058009790 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.069+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0641a47d0 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.070+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc058010430 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.070+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc058018680 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.070+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0641a4cd0 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.071+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fc06410fd50 con 0x7fc0640ffe00 2023-07-14T11:10:39.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.073+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc0580105a0 con 0x7fc0640ffe00 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc061ffb700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0540a1150 0x7fc0540a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fc058014070 con 0x7fc0640ffe00 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc061ffb700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] conn(0x7fc0540ac160 0x7fc0540ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fc0540aecc0 con 0x7fc0540ac160 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc0580cd480 con 0x7fc0640ffe00 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.074+0000 7fc063fff700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0540a1150 0x7fc0540a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.075+0000 7fc0696b2700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] conn(0x7fc0540ac160 0x7fc0540ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.075+0000 7fc063fff700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0540a1150 0x7fc0540a3600 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc064074000 tx=0x7fc050005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.076+0000 7fc0696b2700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] conn(0x7fc0540ac160 0x7fc0540ae5f0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.076+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== osd.5 v2:172.21.15.130:6808/3826515733 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7fc0540aecc0 con 0x7fc0540ac160 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.093+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 --> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fc06404e210 con 0x7fc0540ac160 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.094+0000 7fc061ffb700 1 -- 172.21.15.40:0/2833661206 <== osd.5 v2:172.21.15.130:6808/3826515733 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7fc06404e210 con 0x7fc0540ac160 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.094+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] conn(0x7fc0540ac160 msgr2=0x7fc0540ae5f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.094+0000 7fc06b115700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] conn(0x7fc0540ac160 0x7fc0540ae5f0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0540a1150 msgr2=0x7fc0540a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0540a1150 0x7fc0540a3600 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fc064074000 tx=0x7fc050005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 msgr2=0x7fc064072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 --2- 172.21.15.40:0/2833661206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0640ffe00 0x7fc064072810 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fc05800e9a0 tx=0x7fc05800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc0696b2700 1 -- 172.21.15.40:0/2833661206 reap_dead start 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 shutdown_connections 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 >> 172.21.15.40:0/2833661206 conn(0x7fc0640fb660 msgr2=0x7fc0640fc110 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 shutdown_connections 2023-07-14T11:10:39.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.095+0000 7fc06b115700 1 -- 172.21.15.40:0/2833661206 wait complete. 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 -- 172.21.15.40:0/4207779851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 msgr2=0x7f665c109c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 --2- 172.21.15.40:0/4207779851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c109c00 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f6650009990 tx=0x7f6650009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 -- 172.21.15.40:0/4207779851 shutdown_connections 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 --2- 172.21.15.40:0/4207779851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c10a140 0x7f665c10c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 --2- 172.21.15.40:0/4207779851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f665c100df0 0x7f665c1011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 --2- 172.21.15.40:0/4207779851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c109c00 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 -- 172.21.15.40:0/4207779851 >> 172.21.15.40:0/4207779851 conn(0x7f665c075b70 msgr2=0x7f665c075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 -- 172.21.15.40:0/4207779851 shutdown_connections 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.808+0000 7f6661129700 1 -- 172.21.15.40:0/4207779851 wait complete. 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.810+0000 7f6661129700 1 Processor -- start 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.810+0000 7f6661129700 1 -- start start 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c100df0 0x7f665c19ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c19bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f665c10a140 0x7f665c195ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f665c196130 con 0x7f665c101700 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f665c1962a0 con 0x7f665c10a140 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f6661129700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f665c196460 con 0x7f665c100df0 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f665a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c19bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f665a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c19bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45386/0 (socket says 172.21.15.40:45386) 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f665a59c700 1 -- 172.21.15.40:0/3422490078 learned_addr learned my addr 172.21.15.40:0/3422490078 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f665ad9d700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c100df0 0x7f665c19ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.811+0000 7f665b59e700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f665c10a140 0x7f665c195ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 -- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f665c10a140 msgr2=0x7f665c195ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f665c10a140 0x7f665c195ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 -- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 msgr2=0x7f665c19bf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c19bf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66500095f0 con 0x7f665c100df0 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665a59c700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f665c101700 0x7f665c19bf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:39.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f665ad9d700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c100df0 0x7f665c19ba40 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f664c00e9f0 tx=0x7f664c00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f664c00c9c0 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f665c196740 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f664c010430 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f664c018680 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.812+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f665c1a2a30 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.813+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f665c10fcc0 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.817+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f664c010860 con 0x7f665c100df0 2023-07-14T11:10:39.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.819+0000 7f6643fff700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66440a11d0 0x7f66440a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.819+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f664c014070 con 0x7f665c100df0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.819+0000 7f6643fff700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] conn(0x7f66440ac1e0 0x7f66440ae670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.819+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f66440aed40 con 0x7f66440ac1e0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.819+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f664c0cce50 con 0x7f665c100df0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.820+0000 7f665a59c700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66440a11d0 0x7f66440a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.820+0000 7f665b59e700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] conn(0x7f66440ac1e0 0x7f66440ae670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.820+0000 7f665b59e700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] conn(0x7f66440ac1e0 0x7f66440ae670 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.820+0000 7f665a59c700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66440a11d0 0x7f66440a3680 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f665c10c890 tx=0x7f6650005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.821+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== osd.4 v2:172.21.15.45:6808/2253566846 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f66440aed40 con 0x7f66440ac1e0 2023-07-14T11:10:39.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.840+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 --> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f665c060b90 con 0x7f66440ac1e0 2023-07-14T11:10:39.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.841+0000 7f6643fff700 1 -- 172.21.15.40:0/3422490078 <== osd.4 v2:172.21.15.45:6808/2253566846 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f665c060b90 con 0x7f66440ac1e0 2023-07-14T11:10:39.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.841+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] conn(0x7f66440ac1e0 msgr2=0x7f66440ae670 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.841+0000 7f6661129700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] conn(0x7f66440ac1e0 0x7f66440ae670 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66440a11d0 msgr2=0x7f66440a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66440a11d0 0x7f66440a3680 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f665c10c890 tx=0x7f6650005960 comp rx=0 tx=0).stop 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c100df0 msgr2=0x7f665c19ba40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 --2- 172.21.15.40:0/3422490078 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f665c100df0 0x7f665c19ba40 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f664c00e9f0 tx=0x7f664c00ed00 comp rx=0 tx=0).stop 2023-07-14T11:10:39.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f665b59e700 1 -- 172.21.15.40:0/3422490078 reap_dead start 2023-07-14T11:10:39.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 shutdown_connections 2023-07-14T11:10:39.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 >> 172.21.15.40:0/3422490078 conn(0x7f665c075b70 msgr2=0x7f665c101b10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 shutdown_connections 2023-07-14T11:10:39.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:38.842+0000 7f6661129700 1 -- 172.21.15.40:0/3422490078 wait complete. 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 -- 172.21.15.40:0/1219728271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d00febe0 msgr2=0x7fb4d00fefb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 --2- 172.21.15.40:0/1219728271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d00febe0 0x7fb4d00fefb0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb4c00098f0 tx=0x7fb4c0009c00 comp rx=0 tx=0).stop 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 -- 172.21.15.40:0/1219728271 shutdown_connections 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 --2- 172.21.15.40:0/1219728271 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4d0103f10 0x7fb4d0104340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 --2- 172.21.15.40:0/1219728271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d00febe0 0x7fb4d00fefb0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 --2- 172.21.15.40:0/1219728271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00ff4f0 0x7fb4d01039d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 -- 172.21.15.40:0/1219728271 >> 172.21.15.40:0/1219728271 conn(0x7fb4d0075b70 msgr2=0x7fb4d0075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 -- 172.21.15.40:0/1219728271 shutdown_connections 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.207+0000 7fb4d4872700 1 -- 172.21.15.40:0/1219728271 wait complete. 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.208+0000 7fb4d4872700 1 Processor -- start 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.209+0000 7fb4d4872700 1 -- start start 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.209+0000 7fb4d4872700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 0x7fb4d019c350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.209+0000 7fb4d4872700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4d00ff4f0 0x7fb4d019c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.209+0000 7fb4d4872700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d0103f10 0x7fb4d01a0d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4d4872700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4d01a1310 con 0x7fb4d00ff4f0 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4d4872700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4d01a1480 con 0x7fb4d0103f10 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4d4872700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4d01a1640 con 0x7fb4d00febe0 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 0x7fb4d019c350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4cdd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4d00ff4f0 0x7fb4d019c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 0x7fb4d019c350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33852/0 (socket says 172.21.15.40:33852) 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 -- 172.21.15.40:0/735906399 learned_addr learned my addr 172.21.15.40:0/735906399 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 -- 172.21.15.40:0/735906399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d0103f10 msgr2=0x7fb4d01a0d00 unknown :-1 s=STATE_CONNECTING_RE l=1).mark_down 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb4d0103f10 0x7fb4d01a0d00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 -- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4d00ff4f0 msgr2=0x7fb4d019c890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4d00ff4f0 0x7fb4d019c890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.210+0000 7fb4ce59c700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb4c0009520 con 0x7fb4d00febe0 2023-07-14T11:10:39.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.211+0000 7fb4ce59c700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 0x7fb4d019c350 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb4c0009810 tx=0x7fb4c000fa00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.211+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4c001c070 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.211+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb4d01a18c0 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.211+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb4d01a9180 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.212+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb4c000bbd0 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.212+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb4c0004930 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.212+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fb4b0000ff0 con 0x7fb4d00febe0 2023-07-14T11:10:39.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.215+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fb4c0020070 con 0x7fb4d00febe0 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.216+0000 7fb4bf7fe700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4b80a1150 0x7fb4b80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fb4c00cdda0 con 0x7fb4d00febe0 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4cdd9b700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4b80a1150 0x7fb4b80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4bf7fe700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] conn(0x7fb4b80ac030 0x7fb4b80ae4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fb4b80aeb90 con 0x7fb4b80ac030 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb4c00ce110 con 0x7fb4d00febe0 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.217+0000 7fb4ced9d700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] conn(0x7fb4b80ac030 0x7fb4b80ae4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.218+0000 7fb4cdd9b700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4b80a1150 0x7fb4b80a3600 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fb4d019d970 tx=0x7fb4c4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.218+0000 7fb4ced9d700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] conn(0x7fb4b80ac030 0x7fb4b80ae4c0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.6 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.219+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== osd.6 v2:172.21.15.40:6818/4118960469 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7fb4b80aeb90 con 0x7fb4b80ac030 2023-07-14T11:10:39.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.243+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 --> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fb4b0002d70 con 0x7fb4b80ac030 2023-07-14T11:10:39.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.243+0000 7fb4bf7fe700 1 -- 172.21.15.40:0/735906399 <== osd.6 v2:172.21.15.40:6818/4118960469 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7fb4b0002d70 con 0x7fb4b80ac030 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] conn(0x7fb4b80ac030 msgr2=0x7fb4b80ae4c0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] conn(0x7fb4b80ac030 0x7fb4b80ae4c0 crc :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4b80a1150 msgr2=0x7fb4b80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fb4b80a1150 0x7fb4b80a3600 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7fb4d019d970 tx=0x7fb4c4009260 comp rx=0 tx=0).stop 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 msgr2=0x7fb4d019c350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 --2- 172.21.15.40:0/735906399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb4d00febe0 0x7fb4d019c350 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fb4c0009810 tx=0x7fb4c000fa00 comp rx=0 tx=0).stop 2023-07-14T11:10:39.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4ced9d700 1 -- 172.21.15.40:0/735906399 reap_dead start 2023-07-14T11:10:39.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 shutdown_connections 2023-07-14T11:10:39.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 >> 172.21.15.40:0/735906399 conn(0x7fb4d0075b70 msgr2=0x7fb4d0102a60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 shutdown_connections 2023-07-14T11:10:39.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.244+0000 7fb4d4872700 1 -- 172.21.15.40:0/735906399 wait complete. 2023-07-14T11:10:39.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 -- 172.21.15.40:0/217096567 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe01007a0 msgr2=0x7fdbe0104770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/217096567 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe01007a0 0x7fdbe0104770 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7fdbd0009990 tx=0x7fdbd0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 -- 172.21.15.40:0/217096567 shutdown_connections 2023-07-14T11:10:39.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/217096567 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbe0104ea0 0x7fdbe010ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/217096567 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe00ffe00 0x7fdbe01001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/217096567 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe01007a0 0x7fdbe0104770 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 -- 172.21.15.40:0/217096567 >> 172.21.15.40:0/217096567 conn(0x7fdbe00fb660 msgr2=0x7fdbe00fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 -- 172.21.15.40:0/217096567 shutdown_connections 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.284+0000 7fdbe74ae700 1 -- 172.21.15.40:0/217096567 wait complete. 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.285+0000 7fdbe74ae700 1 Processor -- start 2023-07-14T11:10:39.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.285+0000 7fdbe74ae700 1 -- start start 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbe00ffe00 0x7fdbe019ba10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 0x7fdbe019bf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 0x7fdbe0195a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbe0196100 con 0x7fdbe01007a0 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbe0196270 con 0x7fdbe0104ea0 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe74ae700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbe0196430 con 0x7fdbe00ffe00 2023-07-14T11:10:39.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe4a49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 0x7fdbe019bf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 0x7fdbe0195a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 0x7fdbe0195a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:47984/0 (socket says 172.21.15.40:47984) 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe4a49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 0x7fdbe019bf50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45460/0 (socket says 172.21.15.40:45460) 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 -- 172.21.15.40:0/1883334806 learned_addr learned my addr 172.21.15.40:0/1883334806 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe524a700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbe00ffe00 0x7fdbe019ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 -- 172.21.15.40:0/1883334806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbe00ffe00 msgr2=0x7fdbe019ba10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbe00ffe00 0x7fdbe019ba10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 -- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 msgr2=0x7fdbe019bf50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 0x7fdbe019bf50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe5a4b700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdbd00095f0 con 0x7fdbe0104ea0 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.286+0000 7fdbe4a49700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbe01007a0 0x7fdbe019bf50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbe5a4b700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 0x7fdbe0195a90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fdbd800ef30 tx=0x7fdbd800c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdbd8018070 con 0x7fdbe0104ea0 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdbd800f040 con 0x7fdbe0104ea0 2023-07-14T11:10:39.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdbe01966b0 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdbd800cd00 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.287+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdbe01a2a00 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.288+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fdbe004e210 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.290+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fdbd8003a00 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbd67fc700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbcc0a1150 0x7fdbcc0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fdbd80cd070 con 0x7fdbe0104ea0 2023-07-14T11:10:39.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbe524a700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbcc0a1150 0x7fdbcc0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbd67fc700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] conn(0x7fdbcc0ac160 0x7fdbcc0ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fdbcc0aecc0 con 0x7fdbcc0ac160 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdbd80cd3e0 con 0x7fdbe0104ea0 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbe524a700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbcc0a1150 0x7fdbcc0a3600 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fdbdc005950 tx=0x7fdbdc00a300 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.292+0000 7fdbe4a49700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] conn(0x7fdbcc0ac160 0x7fdbcc0ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.293+0000 7fdbe4a49700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] conn(0x7fdbcc0ac160 0x7fdbcc0ae5f0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.7 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:39.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.294+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== osd.7 v2:172.21.15.45:6816/2329017282 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7fdbcc0aecc0 con 0x7fdbcc0ac160 2023-07-14T11:10:39.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.311+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 --> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fdbe00fca90 con 0x7fdbcc0ac160 2023-07-14T11:10:39.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.311+0000 7fdbd67fc700 1 -- 172.21.15.40:0/1883334806 <== osd.7 v2:172.21.15.45:6816/2329017282 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7fdbe00fca90 con 0x7fdbcc0ac160 2023-07-14T11:10:39.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] conn(0x7fdbcc0ac160 msgr2=0x7fdbcc0ae5f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] conn(0x7fdbcc0ac160 0x7fdbcc0ae5f0 crc :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbcc0a1150 msgr2=0x7fdbcc0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbcc0a1150 0x7fdbcc0a3600 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fdbdc005950 tx=0x7fdbdc00a300 comp rx=0 tx=0).stop 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 msgr2=0x7fdbe0195a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 --2- 172.21.15.40:0/1883334806 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbe0104ea0 0x7fdbe0195a90 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fdbd800ef30 tx=0x7fdbd800c3a0 comp rx=0 tx=0).stop 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe5a4b700 1 -- 172.21.15.40:0/1883334806 reap_dead start 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 shutdown_connections 2023-07-14T11:10:39.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 >> 172.21.15.40:0/1883334806 conn(0x7fdbe00fb660 msgr2=0x7fdbe00fc140 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:39.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 shutdown_connections 2023-07-14T11:10:39.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:39.312+0000 7fdbe74ae700 1 -- 172.21.15.40:0/1883334806 wait complete. 2023-07-14T11:10:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:39 smithi130 ceph-mon[129405]: pgmap v306: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:40.043 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:39 smithi040 ceph-mon[97954]: pgmap v306: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:39 smithi045 ceph-mon[128791]: pgmap v306: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:40.767 INFO:teuthology.orchestra.run.smithi040.stdout:197568495661 2023-07-14T11:10:40.767 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.6 2023-07-14T11:10:40.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 -- 172.21.15.40:0/3275391376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 msgr2=0x7fee24109c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/3275391376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee24109c00 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fee10009990 tx=0x7fee10009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 -- 172.21.15.40:0/3275391376 shutdown_connections 2023-07-14T11:10:40.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/3275391376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee2410a140 0x7fee2410c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/3275391376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee24100df0 0x7fee241011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/3275391376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee24109c00 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 -- 172.21.15.40:0/3275391376 >> 172.21.15.40:0/3275391376 conn(0x7fee24075b70 msgr2=0x7fee24075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:40.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 -- 172.21.15.40:0/3275391376 shutdown_connections 2023-07-14T11:10:40.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.867+0000 7fee2ac6b700 1 -- 172.21.15.40:0/3275391376 wait complete. 2023-07-14T11:10:40.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.868+0000 7fee2ac6b700 1 Processor -- start 2023-07-14T11:10:40.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.868+0000 7fee2ac6b700 1 -- start start 2023-07-14T11:10:40.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 0x7fee240fea50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:40.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee240fef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:40.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2410a140 0x7fee240ff510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:40.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee241a40e0 con 0x7fee24101700 2023-07-14T11:10:40.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee241a4250 con 0x7fee24100df0 2023-07-14T11:10:40.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee2ac6b700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee241a4410 con 0x7fee2410a140 2023-07-14T11:10:40.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee23fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee240fef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:40.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee28a07700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 0x7fee240fea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:40.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee28a07700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 0x7fee240fea50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:47998/0 (socket says 172.21.15.40:47998) 2023-07-14T11:10:40.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee28a07700 1 -- 172.21.15.40:0/4053650413 learned_addr learned my addr 172.21.15.40:0/4053650413 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:40.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee29208700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2410a140 0x7fee240ff510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee23fff700 1 -- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2410a140 msgr2=0x7fee240ff510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee23fff700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fee2410a140 0x7fee240ff510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee23fff700 1 -- 172.21.15.40:0/4053650413 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 msgr2=0x7fee240fea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.869+0000 7fee23fff700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 0x7fee240fea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee23fff700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee100095f0 con 0x7fee24101700 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee28a07700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fee24100df0 0x7fee240fea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:40.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee23fff700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee240fef90 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fee10000c00 tx=0x7fee10005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:40.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee1001d070 con 0x7fee24101700 2023-07-14T11:10:40.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fee241a4690 con 0x7fee24101700 2023-07-14T11:10:40.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fee10021410 con 0x7fee24101700 2023-07-14T11:10:40.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fee10017650 con 0x7fee24101700 2023-07-14T11:10:40.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.870+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fee241a4b30 con 0x7fee24101700 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.871+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fee2404e210 con 0x7fee24101700 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.874+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fee100217f0 con 0x7fee24101700 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee21ffb700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fee140a1150 0x7fee140a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fee100ce8d0 con 0x7fee24101700 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee28a07700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fee140a1150 0x7fee140a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee21ffb700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] conn(0x7fee140ac160 0x7fee140ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:40.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fee140aecc0 con 0x7fee140ac160 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7fee100cec40 con 0x7fee24101700 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.876+0000 7fee29208700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] conn(0x7fee140ac160 0x7fee140ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.877+0000 7fee28a07700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fee140a1150 0x7fee140a3600 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fee18011550 tx=0x7fee18009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.877+0000 7fee29208700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] conn(0x7fee140ac160 0x7fee140ae5f0 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.878+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== osd.2 v2:172.21.15.130:6800/1035271032 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7fee140aecc0 con 0x7fee140ac160 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.895+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 --> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fee2410fd70 con 0x7fee140ac160 2023-07-14T11:10:40.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee21ffb700 1 -- 172.21.15.40:0/4053650413 <== osd.2 v2:172.21.15.130:6800/1035271032 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fee1c00f040 con 0x7fee140ac160 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] conn(0x7fee140ac160 msgr2=0x7fee140ae5f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] conn(0x7fee140ac160 0x7fee140ae5f0 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fee140a1150 msgr2=0x7fee140a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fee140a1150 0x7fee140a3600 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fee18011550 tx=0x7fee18009260 comp rx=0 tx=0).stop 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 msgr2=0x7fee240fef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 --2- 172.21.15.40:0/4053650413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee24101700 0x7fee240fef90 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fee10000c00 tx=0x7fee10005740 comp rx=0 tx=0).stop 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee29208700 1 -- 172.21.15.40:0/4053650413 reap_dead start 2023-07-14T11:10:40.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 shutdown_connections 2023-07-14T11:10:40.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 >> 172.21.15.40:0/4053650413 conn(0x7fee24075b70 msgr2=0x7fee24101b10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:40.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 shutdown_connections 2023-07-14T11:10:40.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:40.896+0000 7fee2ac6b700 1 -- 172.21.15.40:0/4053650413 wait complete. 2023-07-14T11:10:41.006 INFO:teuthology.orchestra.run.smithi040.stdout:34359738442 2023-07-14T11:10:41.006 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.0 2023-07-14T11:10:41.562 INFO:teuthology.orchestra.run.smithi040.stdout:141733920824 2023-07-14T11:10:41.563 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.4 2023-07-14T11:10:41.570 INFO:teuthology.orchestra.run.smithi040.stdout:163208757299 2023-07-14T11:10:41.570 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.5 2023-07-14T11:10:41.573 INFO:teuthology.orchestra.run.smithi040.stdout:219043332136 2023-07-14T11:10:41.573 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.7 2023-07-14T11:10:41.774 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:41 smithi040 ceph-mon[97954]: pgmap v307: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2023-07-14T11:10:41.814 INFO:teuthology.orchestra.run.smithi040.stdout:55834574918 2023-07-14T11:10:41.814 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.1 2023-07-14T11:10:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:41 smithi130 ceph-mon[129405]: pgmap v307: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2023-07-14T11:10:42.109 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:41 smithi045 ceph-mon[128791]: pgmap v307: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 11 op/s 2023-07-14T11:10:42.192 INFO:teuthology.orchestra.run.smithi040.stdout:309237645338 2023-07-14T11:10:42.192 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.10 2023-07-14T11:10:42.506 INFO:teuthology.orchestra.run.smithi040.stdout:77309411393 2023-07-14T11:10:42.506 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.2 2023-07-14T11:10:42.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 -- 172.21.15.40:0/2818998432 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 msgr2=0x7f84181069b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2818998432 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 0x7f84181069b0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f8414009990 tx=0x7f8414009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 -- 172.21.15.40:0/2818998432 shutdown_connections 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2818998432 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 0x7f841810ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2818998432 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 0x7f8418102410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2818998432 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 0x7f84181069b0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.530+0000 7f841fd5d700 1 -- 172.21.15.40:0/2818998432 >> 172.21.15.40:0/2818998432 conn(0x7f84180fd900 msgr2=0x7f84180ffd10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.531+0000 7f841fd5d700 1 -- 172.21.15.40:0/2818998432 shutdown_connections 2023-07-14T11:10:42.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.531+0000 7f841fd5d700 1 -- 172.21.15.40:0/2818998432 wait complete. 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.532+0000 7f841fd5d700 1 Processor -- start 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.532+0000 7f841fd5d700 1 -- start start 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 0x7f8418074a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 0x7f8418074f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 0x7f84180754a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84181a8580 con 0x7f8418102040 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84181a86f0 con 0x7f84181029e0 2023-07-14T11:10:42.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841fd5d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84181a88b0 con 0x7f84181070e0 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841daf9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 0x7f8418074a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841daf9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 0x7f8418074a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45494/0 (socket says 172.21.15.40:45494) 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841daf9700 1 -- 172.21.15.40:0/2602401226 learned_addr learned my addr 172.21.15.40:0/2602401226 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841e2fa700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 0x7f84180754a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.533+0000 7f841d2f8700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 0x7f8418074f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.534+0000 7f841e2fa700 1 -- 172.21.15.40:0/2602401226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 msgr2=0x7f8418074f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.534+0000 7f841e2fa700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f84181029e0 0x7f8418074f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.534+0000 7f841e2fa700 1 -- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 msgr2=0x7f8418074a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.534+0000 7f841e2fa700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8418102040 0x7f8418074a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.534+0000 7f841e2fa700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8408009520 con 0x7f84181070e0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f841e2fa700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 0x7f84180754a0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f841000efc0 tx=0x7f841000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8410018070 con 0x7f84181070e0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84140095f0 con 0x7f84181070e0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f841000f040 con 0x7f84181070e0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f841000cd00 con 0x7f84181070e0 2023-07-14T11:10:42.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.535+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84181a8e70 con 0x7f84181070e0 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.536+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f8418111fe0 con 0x7f84181070e0 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.539+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8410007500 con 0x7f84181070e0 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.540+0000 7f840effd700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84040a12a0 0x7f84040a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.540+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f84100ccf20 con 0x7f84181070e0 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f841daf9700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84040a12a0 0x7f84040a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f840effd700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] conn(0x7f84040ac2b0 0x7f84040ae740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f84040aee10 con 0x7f84040ac2b0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f84100cd2d0 con 0x7f84181070e0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f841d2f8700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] conn(0x7f84040ac2b0 0x7f84040ae740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.541+0000 7f841daf9700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84040a12a0 0x7f84040a3750 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f8408009ef0 tx=0x7f8408009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.542+0000 7f841d2f8700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] conn(0x7f84040ac2b0 0x7f84040ae740 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.9 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.543+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== osd.9 v2:172.21.15.40:6826/1901957249 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f84040aee10 con 0x7f84040ac2b0 2023-07-14T11:10:42.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.561+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 --> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f841804e210 con 0x7f84040ac2b0 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f840effd700 1 -- 172.21.15.40:0/2602401226 <== osd.9 v2:172.21.15.40:6826/1901957249 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f841804e210 con 0x7f84040ac2b0 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] conn(0x7f84040ac2b0 msgr2=0x7f84040ae740 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] conn(0x7f84040ac2b0 0x7f84040ae740 crc :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84040a12a0 msgr2=0x7f84040a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f84040a12a0 0x7f84040a3750 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f8408009ef0 tx=0x7f8408009260 comp rx=0 tx=0).stop 2023-07-14T11:10:42.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 msgr2=0x7f84180754a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 --2- 172.21.15.40:0/2602401226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f84181070e0 0x7f84180754a0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f841000efc0 tx=0x7f841000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841e2fa700 1 -- 172.21.15.40:0/2602401226 reap_dead start 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 shutdown_connections 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.562+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 >> 172.21.15.40:0/2602401226 conn(0x7f84180fd900 msgr2=0x7f84180fe300 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.563+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 shutdown_connections 2023-07-14T11:10:42.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.563+0000 7f841fd5d700 1 -- 172.21.15.40:0/2602401226 wait complete. 2023-07-14T11:10:42.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 -- 172.21.15.40:0/2022172824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 msgr2=0x7f5444102430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 --2- 172.21.15.40:0/2022172824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444102430 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f5434009920 tx=0x7f5434009c30 comp rx=0 tx=0).stop 2023-07-14T11:10:42.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 -- 172.21.15.40:0/2022172824 shutdown_connections 2023-07-14T11:10:42.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 --2- 172.21.15.40:0/2022172824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444107100 0x7f544410ec80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 --2- 172.21.15.40:0/2022172824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444102430 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 --2- 172.21.15.40:0/2022172824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444102a00 0x7f54441069d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.691+0000 7f5449694700 1 -- 172.21.15.40:0/2022172824 >> 172.21.15.40:0/2022172824 conn(0x7f54440fd900 msgr2=0x7f54440ffd10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:42.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.692+0000 7f5449694700 1 -- 172.21.15.40:0/2022172824 shutdown_connections 2023-07-14T11:10:42.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.692+0000 7f5449694700 1 -- 172.21.15.40:0/2022172824 wait complete. 2023-07-14T11:10:42.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.693+0000 7f5449694700 1 Processor -- start 2023-07-14T11:10:42.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.693+0000 7f5449694700 1 -- start start 2023-07-14T11:10:42.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444102a00 0x7f5444198510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444107100 0x7f544419c980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f544419cf90 con 0x7f5444107100 2023-07-14T11:10:42.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f544419d100 con 0x7f5444102a00 2023-07-14T11:10:42.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5449694700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f544419d2c0 con 0x7f5444102060 2023-07-14T11:10:42.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5442ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5442ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444197fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33918/0 (socket says 172.21.15.40:33918) 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.694+0000 7f5442ffd700 1 -- 172.21.15.40:0/3653620399 learned_addr learned my addr 172.21.15.40:0/3653620399 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f5442ffd700 1 -- 172.21.15.40:0/3653620399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444102a00 msgr2=0x7f5444198510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f54437fe700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444107100 0x7f544419c980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f54427fc700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444102a00 0x7f5444198510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f5442ffd700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444102a00 0x7f5444198510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f5442ffd700 1 -- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444107100 msgr2=0x7f544419c980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f5442ffd700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444107100 0x7f544419c980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f5442ffd700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5434009520 con 0x7f5444102060 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.695+0000 7f54437fe700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5444107100 0x7f544419c980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f5442ffd700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444197fd0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f5434009230 tx=0x7f543400fab0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f54427fc700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5444102a00 0x7f5444198510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:10:42.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f543401c070 con 0x7f5444102060 2023-07-14T11:10:42.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f544419d540 con 0x7f5444102060 2023-07-14T11:10:42.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5434005040 con 0x7f5444102060 2023-07-14T11:10:42.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5434004ad0 con 0x7f5444102060 2023-07-14T11:10:42.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.696+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5444074b20 con 0x7f5444102060 2023-07-14T11:10:42.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.697+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f5444111fc0 con 0x7f5444102060 2023-07-14T11:10:42.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.701+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5434020070 con 0x7f5444102060 2023-07-14T11:10:42.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.702+0000 7f542bfff700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f542c0a11d0 0x7f542c0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.702+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f54340cdab0 con 0x7f5444102060 2023-07-14T11:10:42.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.703+0000 7f542bfff700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] conn(0x7f542c0ac160 0x7f542c0ae5f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:42.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.703+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f542c0aecc0 con 0x7f542c0ac160 2023-07-14T11:10:42.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.703+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5434096f20 con 0x7f5444102060 2023-07-14T11:10:42.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.703+0000 7f54427fc700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f542c0a11d0 0x7f542c0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.704+0000 7f54437fe700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] conn(0x7f542c0ac160 0x7f542c0ae5f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:42.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.704+0000 7f54427fc700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f542c0a11d0 0x7f542c0a3680 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f54441995f0 tx=0x7f5438009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.704+0000 7f54437fe700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] conn(0x7f542c0ac160 0x7f542c0ae5f0 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.11 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:42.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.705+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== osd.11 v2:172.21.15.130:6824/692245935 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f542c0aecc0 con 0x7f542c0ac160 2023-07-14T11:10:42.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.725+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 --> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f544419d810 con 0x7f542c0ac160 2023-07-14T11:10:42.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f542bfff700 1 -- 172.21.15.40:0/3653620399 <== osd.11 v2:172.21.15.130:6824/692245935 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f543c011040 con 0x7f542c0ac160 2023-07-14T11:10:42.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] conn(0x7f542c0ac160 msgr2=0x7f542c0ae5f0 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] conn(0x7f542c0ac160 0x7f542c0ae5f0 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f542c0a11d0 msgr2=0x7f542c0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f542c0a11d0 0x7f542c0a3680 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f54441995f0 tx=0x7f5438009260 comp rx=0 tx=0).stop 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 msgr2=0x7f5444197fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.726+0000 7f5449694700 1 --2- 172.21.15.40:0/3653620399 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5444102060 0x7f5444197fd0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f5434009230 tx=0x7f543400fab0 comp rx=0 tx=0).stop 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.727+0000 7f54437fe700 1 -- 172.21.15.40:0/3653620399 reap_dead start 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.727+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 shutdown_connections 2023-07-14T11:10:42.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.727+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 >> 172.21.15.40:0/3653620399 conn(0x7f54440fd900 msgr2=0x7f54440fe410 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:42.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.727+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 shutdown_connections 2023-07-14T11:10:42.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:42.727+0000 7f5449694700 1 -- 172.21.15.40:0/3653620399 wait complete. 2023-07-14T11:10:43.240 INFO:teuthology.orchestra.run.smithi040.stdout:287762808864 2023-07-14T11:10:43.240 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.9 2023-07-14T11:10:43.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 -- 172.21.15.40:0/3712176246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 msgr2=0x7f9f0c104770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 --2- 172.21.15.40:0/3712176246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c104770 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f9f000099c0 tx=0x7f9f00009cd0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 -- 172.21.15.40:0/3712176246 shutdown_connections 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 --2- 172.21.15.40:0/3712176246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f0c104ea0 0x7f9f0c10ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 --2- 172.21.15.40:0/3712176246 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c0ffe00 0x7f9f0c1001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 --2- 172.21.15.40:0/3712176246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c104770 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 -- 172.21.15.40:0/3712176246 >> 172.21.15.40:0/3712176246 conn(0x7f9f0c0fb660 msgr2=0x7f9f0c0fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 -- 172.21.15.40:0/3712176246 shutdown_connections 2023-07-14T11:10:43.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.335+0000 7f9f123d3700 1 -- 172.21.15.40:0/3712176246 wait complete. 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.336+0000 7f9f123d3700 1 Processor -- start 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 -- start start 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f0c0ffe00 0x7f9f0c195cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c196200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c104ea0 0x7f9f0c1995c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c199bc0 con 0x7f9f0c0ffe00 2023-07-14T11:10:43.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c199d30 con 0x7f9f0c104ea0 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f123d3700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f0c199ef0 con 0x7f9f0c1007a0 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f0bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f0c0ffe00 0x7f9f0c195cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f0b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c196200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f0b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c196200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33938/0 (socket says 172.21.15.40:33938) 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.337+0000 7f9f0b7fe700 1 -- 172.21.15.40:0/818529797 learned_addr learned my addr 172.21.15.40:0/818529797 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:43.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f10970700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c104ea0 0x7f9f0c1995c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 -- 172.21.15.40:0/818529797 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c104ea0 msgr2=0x7f9f0c1995c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c104ea0 0x7f9f0c1995c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 -- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f0c0ffe00 msgr2=0x7f9f0c195cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f0c0ffe00 0x7f9f0c195cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f000095f0 con 0x7f9f0c1007a0 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f10970700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9f0c104ea0 0x7f9f0c1995c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f0b7fe700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c196200 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f9f000098e0 tx=0x7f9f0000f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:43.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f0001d070 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9f0000fb80 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f0c19a170 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.338+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9f000173f0 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.339+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f0c1a2b20 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.339+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f9f0c10fd20 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.342+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9f0000fcf0 con 0x7f9f0c1007a0 2023-07-14T11:10:43.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.343+0000 7f9f097fa700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ef40a11d0 0x7f9ef40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.343+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f9f000cd920 con 0x7f9f0c1007a0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.343+0000 7f9f0bfff700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ef40a11d0 0x7f9ef40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.343+0000 7f9f097fa700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] conn(0x7f9ef40ac1e0 0x7f9ef40ae670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.343+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f9ef40aed40 con 0x7f9ef40ac1e0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.344+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_get_version_reply(handle=1 version=90) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9f000cdc90 con 0x7f9f0c1007a0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.344+0000 7f9f10970700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] conn(0x7f9ef40ac1e0 0x7f9ef40ae670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.344+0000 7f9f0bfff700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ef40a11d0 0x7f9ef40a3680 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f9efc009c60 tx=0x7f9efc009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:43.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.345+0000 7f9f10970700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] conn(0x7f9ef40ac1e0 0x7f9ef40ae670 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.8 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:43.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.346+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== osd.8 v2:172.21.15.130:6816/1753746487 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+25106 (crc 0 0 0) 0x7f9ef40aed40 con 0x7f9ef40ac1e0 2023-07-14T11:10:43.365 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.363+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 --> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f9f0c19a490 con 0x7f9ef40ac1e0 2023-07-14T11:10:43.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.363+0000 7f9f097fa700 1 -- 172.21.15.40:0/818529797 <== osd.8 v2:172.21.15.130:6816/1753746487 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f9f0400f040 con 0x7f9ef40ac1e0 2023-07-14T11:10:43.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] conn(0x7f9ef40ac1e0 msgr2=0x7f9ef40ae670 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] conn(0x7f9ef40ac1e0 0x7f9ef40ae670 crc :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:43.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ef40a11d0 msgr2=0x7f9ef40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9ef40a11d0 0x7f9ef40a3680 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f9efc009c60 tx=0x7f9efc009260 comp rx=0 tx=0).stop 2023-07-14T11:10:43.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 msgr2=0x7f9f0c196200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 --2- 172.21.15.40:0/818529797 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9f0c1007a0 0x7f9f0c196200 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f9f000098e0 tx=0x7f9f0000f690 comp rx=0 tx=0).stop 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f10970700 1 -- 172.21.15.40:0/818529797 reap_dead start 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 shutdown_connections 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 >> 172.21.15.40:0/818529797 conn(0x7f9f0c0fb660 msgr2=0x7f9f0c0fc0f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 shutdown_connections 2023-07-14T11:10:43.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:43.364+0000 7f9f123d3700 1 -- 172.21.15.40:0/818529797 wait complete. 2023-07-14T11:10:43.962 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:43 smithi040 ceph-mon[97954]: pgmap v308: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 10 op/s 2023-07-14T11:10:43.965 INFO:teuthology.orchestra.run.smithi040.stdout:253403070501 2023-07-14T11:10:43.965 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.8 2023-07-14T11:10:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:43 smithi130 ceph-mon[129405]: pgmap v308: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 10 op/s 2023-07-14T11:10:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:43 smithi045 ceph-mon[128791]: pgmap v308: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 10 op/s 2023-07-14T11:10:44.237 INFO:teuthology.orchestra.run.smithi040.stdout:339302416409 2023-07-14T11:10:44.238 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph osd last-stat-seq osd.11 2023-07-14T11:10:44.435 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.447 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.546 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.601 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.604 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.711 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.713 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:44.861 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:45.468 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:45 smithi040 ceph-mon[97954]: pgmap v309: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:10:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:45 smithi130 ceph-mon[129405]: pgmap v309: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:10:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:45 smithi045 ceph-mon[128791]: pgmap v309: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:10:46.925 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:47.025 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:47.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:47 smithi040 ceph-mon[97954]: pgmap v310: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:47.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:47 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:10:48.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:47 smithi130 ceph-mon[129405]: pgmap v310: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:47 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:10:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:47 smithi045 ceph-mon[128791]: pgmap v310: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:10:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:47 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.723+0000 7f6f462b9700 1 -- 172.21.15.40:0/823629529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f40101570 msgr2=0x7f6f40109c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.723+0000 7f6f462b9700 1 --2- 172.21.15.40:0/823629529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f40101570 0x7f6f40109c40 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f6f34009990 tx=0x7f6f34009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 -- 172.21.15.40:0/823629529 shutdown_connections 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 --2- 172.21.15.40:0/823629529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f4010a180 0x7f6f4010c5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 --2- 172.21.15.40:0/823629529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40100c60 0x7f6f40101030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 --2- 172.21.15.40:0/823629529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f40101570 0x7f6f40109c40 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 -- 172.21.15.40:0/823629529 >> 172.21.15.40:0/823629529 conn(0x7f6f40075b70 msgr2=0x7f6f40075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:48.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 -- 172.21.15.40:0/823629529 shutdown_connections 2023-07-14T11:10:48.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.724+0000 7f6f462b9700 1 -- 172.21.15.40:0/823629529 wait complete. 2023-07-14T11:10:48.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.725+0000 7f6f462b9700 1 Processor -- start 2023-07-14T11:10:48.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.725+0000 7f6f462b9700 1 -- start start 2023-07-14T11:10:48.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:48.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40101570 0x7f6f400fefd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:48.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4010a180 0x7f6f400ff550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f401a4120 con 0x7f6f4010a180 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f401a4290 con 0x7f6f40100c60 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f462b9700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f401a4450 con 0x7f6f40101570 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40101570 0x7f6f400fefd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56936/0 (socket says 172.21.15.40:56936) 2023-07-14T11:10:48.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 -- 172.21.15.40:0/3927619954 learned_addr learned my addr 172.21.15.40:0/3927619954 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f44856700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4010a180 0x7f6f400ff550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 -- 172.21.15.40:0/3927619954 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40101570 msgr2=0x7f6f400fefd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40101570 0x7f6f400fefd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 -- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4010a180 msgr2=0x7f6f400ff550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4010a180 0x7f6f400ff550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.726+0000 7f6f3ffff700 1 -- 172.21.15.40:0/3927619954 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f340095f0 con 0x7f6f40100c60 2023-07-14T11:10:48.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f3ffff700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f6f3000e9a0 tx=0x7f6f3000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:48.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f30009790 con 0x7f6f40100c60 2023-07-14T11:10:48.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6f3000cbd0 con 0x7f6f40100c60 2023-07-14T11:10:48.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f401a4730 con 0x7f6f40100c60 2023-07-14T11:10:48.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6f30018680 con 0x7f6f40100c60 2023-07-14T11:10:48.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.727+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f401a4c80 con 0x7f6f40100c60 2023-07-14T11:10:48.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.728+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f4004e210 con 0x7f6f40100c60 2023-07-14T11:10:48.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.730+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6f3000cd40 con 0x7f6f40100c60 2023-07-14T11:10:48.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.733+0000 7f6f3d7fa700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 0x7f6f280a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:48.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.733+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f6f30014070 con 0x7f6f40100c60 2023-07-14T11:10:48.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.733+0000 7f6f3f7fe700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 0x7f6f280a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:48.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.734+0000 7f6f3f7fe700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 0x7f6f280a3680 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6f40100280 tx=0x7f6f34005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:48.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.734+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6f30096300 con 0x7f6f40100c60 2023-07-14T11:10:48.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.946+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f6f400fffc0 con 0x7f6f40100c60 2023-07-14T11:10:48.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.946+0000 7f6f3d7fa700 1 -- 172.21.15.40:0/3927619954 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f6f30099920 con 0x7f6f40100c60 2023-07-14T11:10:48.949 INFO:teuthology.orchestra.run.smithi040.stdout:107374182461 2023-07-14T11:10:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 msgr2=0x7f6f280a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 0x7f6f280a3680 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f6f40100280 tx=0x7f6f34005fb0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 msgr2=0x7f6f400fea90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f6f3000e9a0 tx=0x7f6f3000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 shutdown_connections 2023-07-14T11:10:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6f280a11d0 0x7f6f280a3680 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4010a180 0x7f6f400ff550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6f40100c60 0x7f6f400fea90 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 --2- 172.21.15.40:0/3927619954 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6f40101570 0x7f6f400fefd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 >> 172.21.15.40:0/3927619954 conn(0x7f6f40075b70 msgr2=0x7f6f400fd310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 shutdown_connections 2023-07-14T11:10:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:48.949+0000 7f6f462b9700 1 -- 172.21.15.40:0/3927619954 wait complete. 2023-07-14T11:10:49.629 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:49.906 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: pgmap v311: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:49.906 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3927619954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-07-14T11:10:49.906 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:49.907 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:49.907 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:49.907 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:49 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: pgmap v311: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3927619954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-07-14T11:10:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:49 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: pgmap v311: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3927619954' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-07-14T11:10:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:49 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:10:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 -- 172.21.15.40:0/133797306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d010a140 msgr2=0x7fa1d010c5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 --2- 172.21.15.40:0/133797306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d010a140 0x7fa1d010c5a0 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fa1c800b0a0 tx=0x7fa1c800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 -- 172.21.15.40:0/133797306 shutdown_connections 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 --2- 172.21.15.40:0/133797306 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d010a140 0x7fa1d010c5a0 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 --2- 172.21.15.40:0/133797306 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d0100d60 0x7fa1d0101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 --2- 172.21.15.40:0/133797306 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0109b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 -- 172.21.15.40:0/133797306 >> 172.21.15.40:0/133797306 conn(0x7fa1d00fa5a0 msgr2=0x7fa1d00fc9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 -- 172.21.15.40:0/133797306 shutdown_connections 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.657+0000 7fa1d692d700 1 -- 172.21.15.40:0/133797306 wait complete. 2023-07-14T11:10:50.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.658+0000 7fa1d692d700 1 Processor -- start 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 -- start start 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0196250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 0x7fa1d019a6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1d019acd0 con 0x7fa1d0101670 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1d019ae40 con 0x7fa1d010a140 2023-07-14T11:10:50.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.659+0000 7fa1d692d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa1d019b000 con 0x7fa1d0100d60 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49028/0 (socket says 172.21.15.40:49028) 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cf7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0196250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 -- 172.21.15.40:0/1617811385 learned_addr learned my addr 172.21.15.40:0/1617811385 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1d4eca700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 0x7fa1d019a6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 -- 172.21.15.40:0/1617811385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 msgr2=0x7fa1d019a6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 0x7fa1d019a6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 -- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 msgr2=0x7fa1d0196250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0196250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 -- 172.21.15.40:0/1617811385 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1c0009520 con 0x7fa1d0100d60 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1d4eca700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 0x7fa1d019a6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cf7fe700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0196250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.660+0000 7fa1cffff700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fa1c000eb30 tx=0x7fa1c000ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.661+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1c0009880 con 0x7fa1d0100d60 2023-07-14T11:10:50.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.661+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa1c00044e0 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.661+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1c4009520 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.661+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1c0003d80 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.661+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1c8009d00 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.662+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa1d00681a0 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.665+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa1c0003f20 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.667+0000 7fa1cd7fa700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 0x7fa1b80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.667+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fa1c0014070 con 0x7fa1d0100d60 2023-07-14T11:10:50.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.667+0000 7fa1cf7fe700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 0x7fa1b80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.668+0000 7fa1cf7fe700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 0x7fa1b80a3600 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fa1d0197330 tx=0x7fa1c4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:50.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.670+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa1c0096590 con 0x7fa1d0100d60 2023-07-14T11:10:50.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/427548731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-07-14T11:10:50.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.927+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) v1 -- 0x7fa1d0197070 con 0x7fa1d0100d60 2023-07-14T11:10:50.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 -- 172.21.15.40:0/3843737744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 msgr2=0x7f0fb0101010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/3843737744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb0101010 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f0fa0009990 tx=0x7f0fa0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 -- 172.21.15.40:0/3843737744 shutdown_connections 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/3843737744 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb010a160 0x7f0fb010c5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/3843737744 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb0101010 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/3843737744 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0fb0101550 0x7f0fb0109c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 -- 172.21.15.40:0/3843737744 >> 172.21.15.40:0/3843737744 conn(0x7f0fb0075b70 msgr2=0x7f0fb0075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 -- 172.21.15.40:0/3843737744 shutdown_connections 2023-07-14T11:10:50.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.358+0000 7f0fb5a13700 1 -- 172.21.15.40:0/3843737744 wait complete. 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 Processor -- start 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 -- start start 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb01097a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 0x7f0fb01067c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0fb010a160 0x7f0fb0106d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fb0107300 con 0x7f0fb0100c40 2023-07-14T11:10:50.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fb0107470 con 0x7f0fb010a160 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.360+0000 7f0fb5a13700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0fb0107630 con 0x7f0fb0101550 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0faeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb01097a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0faeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb01097a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42052/0 (socket says 172.21.15.40:42052) 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0faeffd700 1 -- 172.21.15.40:0/427548731 learned_addr learned my addr 172.21.15.40:0/427548731 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 0x7f0fb01067c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 -- 172.21.15.40:0/427548731 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0fb010a160 msgr2=0x7f0fb0106d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0fb010a160 0x7f0fb0106d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 -- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 msgr2=0x7f0fb01097a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb01097a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 -- 172.21.15.40:0/427548731 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0fa00095f0 con 0x7f0fb0101550 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.361+0000 7f0fae7fc700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 0x7f0fb01067c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0fa400e9f0 tx=0x7f0fa400ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.362+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fa40096d0 con 0x7f0fb0101550 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.362+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0fb01a8ae0 con 0x7f0fb0101550 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.362+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0fb01a8fa0 con 0x7f0fb0101550 2023-07-14T11:10:50.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.362+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0fa400cbd0 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.362+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0fa4018680 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.364+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f9c005320 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.367+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f0fa400cd40 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.369+0000 7f0fb4a11700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 0x7f0f980a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.369+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f0fa4014070 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.369+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0fa4099c70 con 0x7f0fb0101550 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.369+0000 7f0faeffd700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 0x7f0f980a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:50.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.371+0000 7f0faeffd700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 0x7f0f980a3680 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f0fa0006010 tx=0x7f0fa0005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:50.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.603+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7f0f9c005190 con 0x7f0fb0101550 2023-07-14T11:10:50.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.603+0000 7f0fb4a11700 1 -- 172.21.15.40:0/427548731 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f0fa4099800 con 0x7f0fb0101550 2023-07-14T11:10:50.936 INFO:teuthology.orchestra.run.smithi040.stdout:34359738444 2023-07-14T11:10:50.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 msgr2=0x7f0f980a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 0x7f0f980a3680 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f0fa0006010 tx=0x7f0fa0005960 comp rx=0 tx=0).stop 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 msgr2=0x7f0fb01067c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 0x7f0fb01067c0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0fa400e9f0 tx=0x7f0fa400ed00 comp rx=0 tx=0).stop 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 shutdown_connections 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f0f980a11d0 0x7f0f980a3680 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0fb010a160 0x7f0fb0106d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0fb0100c40 0x7f0fb01097a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 --2- 172.21.15.40:0/427548731 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0fb0101550 0x7f0fb01067c0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 >> 172.21.15.40:0/427548731 conn(0x7f0fb0075b70 msgr2=0x7f0fb00fd380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 shutdown_connections 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.606+0000 7f0fb5a13700 1 -- 172.21.15.40:0/427548731 wait complete. 2023-07-14T11:10:50.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.928+0000 7fa1cd7fa700 1 -- 172.21.15.40:0/1617811385 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7fa1c0094e60 con 0x7fa1d0100d60 2023-07-14T11:10:50.939 INFO:teuthology.orchestra.run.smithi040.stdout:163208757301 2023-07-14T11:10:50.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 msgr2=0x7fa1b80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 0x7fa1b80a3600 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fa1d0197330 tx=0x7fa1c4009260 comp rx=0 tx=0).stop 2023-07-14T11:10:50.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 msgr2=0x7fa1d0195d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fa1c000eb30 tx=0x7fa1c000ee40 comp rx=0 tx=0).stop 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 shutdown_connections 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa1b80a1150 0x7fa1b80a3600 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1d010a140 0x7fa1d019a6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa1d0100d60 0x7fa1d0195d10 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 --2- 172.21.15.40:0/1617811385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1d0101670 0x7fa1d0196250 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.930+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 >> 172.21.15.40:0/1617811385 conn(0x7fa1d00fa5a0 msgr2=0x7fa1d010c2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:50.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.931+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 shutdown_connections 2023-07-14T11:10:50.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:50.931+0000 7fa1d692d700 1 -- 172.21.15.40:0/1617811385 wait complete. 2023-07-14T11:10:51.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/427548731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-07-14T11:10:51.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/427548731' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-07-14T11:10:51.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 -- 172.21.15.40:0/1791496193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400fe980 msgr2=0x7f4f400fed50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 --2- 172.21.15.40:0/1791496193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400fe980 0x7f4f400fed50 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f4f30009990 tx=0x7f4f30009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 -- 172.21.15.40:0/1791496193 shutdown_connections 2023-07-14T11:10:51.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 --2- 172.21.15.40:0/1791496193 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f4010c530 0x7f4f4010e910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.198 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 --2- 172.21.15.40:0/1791496193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400fe980 0x7f4f400fed50 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 --2- 172.21.15.40:0/1791496193 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f400ff290 0x7f4f4010bff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 -- 172.21.15.40:0/1791496193 >> 172.21.15.40:0/1791496193 conn(0x7f4f40075b70 msgr2=0x7f4f40075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 -- 172.21.15.40:0/1791496193 shutdown_connections 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.021+0000 7f4f46884700 1 -- 172.21.15.40:0/1791496193 wait complete. 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.022+0000 7f4f46884700 1 Processor -- start 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 -- start start 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.199 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400ff290 0x7f4f40198410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f4010c530 0x7f4f4019c880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f4019ce90 con 0x7f4f400ff290 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f4019d000 con 0x7f4f4010c530 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.023+0000 7f4f46884700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f4019d1c0 con 0x7f4f400fe980 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400ff290 0x7f4f40198410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.200 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49054/0 (socket says 172.21.15.40:49054) 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 -- 172.21.15.40:0/3841548883 learned_addr learned my addr 172.21.15.40:0/3841548883 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f44e21700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f4010c530 0x7f4f4019c880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 -- 172.21.15.40:0/3841548883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f4010c530 msgr2=0x7f4f4019c880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f4010c530 0x7f4f4019c880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 -- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400ff290 msgr2=0x7f4f40198410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400ff290 0x7f4f40198410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 -- 172.21.15.40:0/3841548883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f300095f0 con 0x7f4f400fe980 2023-07-14T11:10:51.201 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.024+0000 7f4f3ffff700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f4f30005910 tx=0x7f4f30005740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.025+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f3001d070 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.025+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f4019d440 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.025+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4f30022470 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.025+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f30017650 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.025+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f401a4d00 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.027+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f4004e210 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.029+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4f3000fad0 con 0x7f4f400fe980 2023-07-14T11:10:51.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.030+0000 7f4f3d7fa700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 0x7f4f280a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.031+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f4f300ce070 con 0x7f4f400fe980 2023-07-14T11:10:51.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.031+0000 7f4f3f7fe700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 0x7f4f280a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.032+0000 7f4f3f7fe700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 0x7f4f280a3600 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4f401994f0 tx=0x7f4f34009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:51.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.034+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4f30097540 con 0x7f4f400fe980 2023-07-14T11:10:51.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.272+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 10} v 0) v1 -- 0x7f4f401a4e90 con 0x7f4f400fe980 2023-07-14T11:10:51.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.272+0000 7f4f3d7fa700 1 -- 172.21.15.40:0/3841548883 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 10}]=0 v0) v1 ==== 75+0+13 (secure 0 0 0) 0x7f4f3009ab60 con 0x7f4f400fe980 2023-07-14T11:10:51.275 INFO:teuthology.orchestra.run.smithi040.stdout:309237645341 2023-07-14T11:10:51.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 msgr2=0x7f4f280a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 0x7f4f280a3600 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f4f401994f0 tx=0x7f4f34009260 comp rx=0 tx=0).stop 2023-07-14T11:10:51.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 msgr2=0x7f4f40197ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f4f30005910 tx=0x7f4f30005740 comp rx=0 tx=0).stop 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 shutdown_connections 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4f280a1150 0x7f4f280a3600 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f4010c530 0x7f4f4019c880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f400fe980 0x7f4f40197ed0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 --2- 172.21.15.40:0/3841548883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f400ff290 0x7f4f40198410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 >> 172.21.15.40:0/3841548883 conn(0x7f4f40075b70 msgr2=0x7f4f400fc3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:51.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 shutdown_connections 2023-07-14T11:10:51.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.275+0000 7f4f46884700 1 -- 172.21.15.40:0/3841548883 wait complete. 2023-07-14T11:10:51.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 -- 172.21.15.40:0/560171003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 msgr2=0x7f20e010ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 --2- 172.21.15.40:0/560171003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e010ca20 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f20d800b0a0 tx=0x7f20d800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 -- 172.21.15.40:0/560171003 shutdown_connections 2023-07-14T11:10:51.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 --2- 172.21.15.40:0/560171003 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e010ca20 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 --2- 172.21.15.40:0/560171003 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 0x7f20e01001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 --2- 172.21.15.40:0/560171003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 0x7f20e0104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 -- 172.21.15.40:0/560171003 >> 172.21.15.40:0/560171003 conn(0x7f20e00fb660 msgr2=0x7f20e00fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 -- 172.21.15.40:0/560171003 shutdown_connections 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.761+0000 7f20e64d4700 1 -- 172.21.15.40:0/560171003 wait complete. 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 Processor -- start 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 -- start start 2023-07-14T11:10:51.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 0x7f20e0197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 0x7f20e01984f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20e019cf70 con 0x7f20e01007a0 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20e019d0e0 con 0x7f20e0104ea0 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.762+0000 7f20e64d4700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f20e019d2a0 con 0x7f20e00ffe00 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20df7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 0x7f20e01984f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:57006/0 (socket says 172.21.15.40:57006) 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 -- 172.21.15.40:0/1969012019 learned_addr learned my addr 172.21.15.40:0/1969012019 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20dffff700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 0x7f20e0197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 -- 172.21.15.40:0/1969012019 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 msgr2=0x7f20e0197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 0x7f20e0197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 -- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 msgr2=0x7f20e01984f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 0x7f20e01984f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 -- 172.21.15.40:0/1969012019 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f20d4009520 con 0x7f20e0104ea0 2023-07-14T11:10:51.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.763+0000 7f20e4a71700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f20d8009fd0 tx=0x7f20d80077f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.764+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d800e040 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.764+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f20d8009d00 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.764+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f20d80094c0 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.764+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f20d80052a0 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.764+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f20e01a4c10 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.765+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f20e004e210 con 0x7f20e0104ea0 2023-07-14T11:10:51.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.767+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f20d801d470 con 0x7f20e0104ea0 2023-07-14T11:10:51.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.768+0000 7f20dd7fa700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 0x7f20c80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:51.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.768+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f20d80ce120 con 0x7f20e0104ea0 2023-07-14T11:10:51.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.769+0000 7f20dffff700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 0x7f20c80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:51.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.769+0000 7f20dffff700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 0x7f20c80a3600 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f20d0009c60 tx=0x7f20d0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:51.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.771+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f20d8097590 con 0x7f20e0104ea0 2023-07-14T11:10:51.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.994+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) v1 -- 0x7f20e01a4e40 con 0x7f20e0104ea0 2023-07-14T11:10:51.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.994+0000 7f20dd7fa700 1 -- 172.21.15.40:0/1969012019 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f20d8095e60 con 0x7f20e0104ea0 2023-07-14T11:10:51.997 INFO:teuthology.orchestra.run.smithi040.stdout:141733920826 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 msgr2=0x7f20c80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 0x7f20c80a3600 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f20d0009c60 tx=0x7f20d0009260 comp rx=0 tx=0).stop 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 msgr2=0x7f20e019c960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f20d8009fd0 tx=0x7f20d80077f0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 shutdown_connections 2023-07-14T11:10:52.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f20c80a1150 0x7f20c80a3600 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f20e0104ea0 0x7f20e019c960 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f20e00ffe00 0x7f20e0197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 --2- 172.21.15.40:0/1969012019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f20e01007a0 0x7f20e01984f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 >> 172.21.15.40:0/1969012019 conn(0x7f20e00fb660 msgr2=0x7f20e00fc060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 shutdown_connections 2023-07-14T11:10:52.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.997+0000 7f20e64d4700 1 -- 172.21.15.40:0/1969012019 wait complete. 2023-07-14T11:10:52.026 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:51 smithi040 ceph-mon[97954]: pgmap v312: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:52.026 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:51 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1617811385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-07-14T11:10:52.026 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:51 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3841548883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2023-07-14T11:10:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:51 smithi130 ceph-mon[129405]: pgmap v312: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:51 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1617811385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-07-14T11:10:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:51 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3841548883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2023-07-14T11:10:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:51 smithi045 ceph-mon[128791]: pgmap v312: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:10:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:51 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1617811385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-07-14T11:10:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:51 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3841548883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 10}]: dispatch 2023-07-14T11:10:52.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.921+0000 7f28f6702700 1 -- 172.21.15.40:0/1415146629 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f00728b0 msgr2=0x7f28f0072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.921+0000 7f28f6702700 1 --2- 172.21.15.40:0/1415146629 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f00728b0 0x7f28f0072c80 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f28e0009990 tx=0x7f28e0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 -- 172.21.15.40:0/1415146629 shutdown_connections 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 --2- 172.21.15.40:0/1415146629 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f010c3e0 0x7f28f010e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 --2- 172.21.15.40:0/1415146629 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f00728b0 0x7f28f0072c80 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 --2- 172.21.15.40:0/1415146629 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f010bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 -- 172.21.15.40:0/1415146629 >> 172.21.15.40:0/1415146629 conn(0x7f28f00fb660 msgr2=0x7f28f00fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:52.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 -- 172.21.15.40:0/1415146629 shutdown_connections 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.922+0000 7f28f6702700 1 -- 172.21.15.40:0/1415146629 wait complete. 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 Processor -- start 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 -- start start 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 0x7f28f0197ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f010c3e0 0x7f28f019c870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f019cf10 con 0x7f28f010c3e0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f019d080 con 0x7f28f00731c0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.923+0000 7f28f6702700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28f019d240 con 0x7f28f00728b0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28f4c9f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f010c3e0 0x7f28f019c870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28effff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 0x7f28f0197ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:57018/0 (socket says 172.21.15.40:57018) 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 -- 172.21.15.40:0/3824096162 learned_addr learned my addr 172.21.15.40:0/3824096162 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:52.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 -- 172.21.15.40:0/3824096162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 msgr2=0x7f28f0197ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 0x7f28f0197ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 -- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f010c3e0 msgr2=0x7f28f019c870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f010c3e0 0x7f28f019c870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 -- 172.21.15.40:0/3824096162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f28e00095f0 con 0x7f28f00731c0 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28effff700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 0x7f28f0197ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.924+0000 7f28ef7fe700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f28e400e9a0 tx=0x7f28e400ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.925+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e4009790 con 0x7f28f00731c0 2023-07-14T11:10:52.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.925+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f28e4010430 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.925+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28f019d520 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.925+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f28e4018680 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.925+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28f01a4cf0 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.926+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f28f004e210 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.928+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f28e40105a0 con 0x7f28f00731c0 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.929+0000 7f28ed7fa700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 0x7f28d80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.929+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f28e4014070 con 0x7f28f00731c0 2023-07-14T11:10:52.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.929+0000 7f28effff700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 0x7f28d80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.930+0000 7f28effff700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 0x7f28d80a3600 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f28e000b2e0 tx=0x7f28e0005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:52.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:51.933+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f28e4051790 con 0x7f28f00731c0 2023-07-14T11:10:52.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.174+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f28f01a4f80 con 0x7f28f00731c0 2023-07-14T11:10:52.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.174+0000 7f28ed7fa700 1 -- 172.21.15.40:0/3824096162 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f28e4096220 con 0x7f28f00731c0 2023-07-14T11:10:52.177 INFO:teuthology.orchestra.run.smithi040.stdout:55834574920 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 msgr2=0x7f28d80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 0x7f28d80a3600 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f28e000b2e0 tx=0x7f28e0005fd0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 msgr2=0x7f28f0198400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f28e400e9a0 tx=0x7f28e400ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 shutdown_connections 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f28d80a1150 0x7f28d80a3600 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28f010c3e0 0x7f28f019c870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f28f00728b0 0x7f28f0197ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 --2- 172.21.15.40:0/3824096162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f28f00731c0 0x7f28f0198400 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 >> 172.21.15.40:0/3824096162 conn(0x7f28f00fb660 msgr2=0x7f28f010e4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:52.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 shutdown_connections 2023-07-14T11:10:52.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.177+0000 7f28f6702700 1 -- 172.21.15.40:0/3824096162 wait complete. 2023-07-14T11:10:52.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.284+0000 7f7bdb458700 1 -- 172.21.15.40:0/4119900611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd4104ea0 msgr2=0x7f7bd410ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.284+0000 7f7bdb458700 1 --2- 172.21.15.40:0/4119900611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd4104ea0 0x7f7bd410ca20 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7f7bcc00b0d0 tx=0x7f7bcc00b3e0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 -- 172.21.15.40:0/4119900611 shutdown_connections 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 --2- 172.21.15.40:0/4119900611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd4104ea0 0x7f7bd410ca20 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 --2- 172.21.15.40:0/4119900611 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd40ffe00 0x7f7bd41001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 --2- 172.21.15.40:0/4119900611 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd41007a0 0x7f7bd4104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 -- 172.21.15.40:0/4119900611 >> 172.21.15.40:0/4119900611 conn(0x7f7bd40fb660 msgr2=0x7f7bd40fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:52.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 -- 172.21.15.40:0/4119900611 shutdown_connections 2023-07-14T11:10:52.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.285+0000 7f7bdb458700 1 -- 172.21.15.40:0/4119900611 wait complete. 2023-07-14T11:10:52.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 Processor -- start 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 -- start start 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 0x7f7bd4072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd4104ea0 0x7f7bd40732d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bd41a41c0 con 0x7f7bd41007a0 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bd41a4330 con 0x7f7bd4104ea0 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.286+0000 7f7bdb458700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7bd41a44f0 con 0x7f7bd40ffe00 2023-07-14T11:10:52.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd89f3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 0x7f7bd4072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd89f3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42138/0 (socket says 172.21.15.40:42138) 2023-07-14T11:10:52.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd89f3700 1 -- 172.21.15.40:0/1114340844 learned_addr learned my addr 172.21.15.40:0/1114340844 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:52.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd99f5700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd4104ea0 0x7f7bd40732d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 -- 172.21.15.40:0/1114340844 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd4104ea0 msgr2=0x7f7bd40732d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd4104ea0 0x7f7bd40732d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 -- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 msgr2=0x7f7bd4072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd91f4700 1 -- 172.21.15.40:0/1114340844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7bcc009d00 con 0x7f7bd40ffe00 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.287+0000 7f7bd89f3700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bd91f4700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 0x7f7bd4072810 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f7bd000e9f0 tx=0x7f7bd000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bd000c9c0 con 0x7f7bd40ffe00 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7bd0010430 con 0x7f7bd40ffe00 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7bd41a47d0 con 0x7f7bd40ffe00 2023-07-14T11:10:52.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7bd0018680 con 0x7f7bd40ffe00 2023-07-14T11:10:52.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.288+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7bd41a4cd0 con 0x7f7bd40ffe00 2023-07-14T11:10:52.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.289+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7bd404e210 con 0x7f7bd40ffe00 2023-07-14T11:10:52.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.292+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f7bd0010860 con 0x7f7bd40ffe00 2023-07-14T11:10:52.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.294+0000 7f7bca7fc700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 0x7f7bc00a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:52.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.294+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f7bd0014070 con 0x7f7bd40ffe00 2023-07-14T11:10:52.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.294+0000 7f7bd89f3700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 0x7f7bc00a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:52.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.295+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f7bd0096130 con 0x7f7bd40ffe00 2023-07-14T11:10:52.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.295+0000 7f7bd89f3700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 0x7f7bc00a3680 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f7bd4074000 tx=0x7f7bc4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:52.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.519+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 7} v 0) v1 -- 0x7f7bd41a5020 con 0x7f7bd40ffe00 2023-07-14T11:10:52.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.520+0000 7f7bca7fc700 1 -- 172.21.15.40:0/1114340844 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 7}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f7bd0099750 con 0x7f7bd40ffe00 2023-07-14T11:10:52.523 INFO:teuthology.orchestra.run.smithi040.stdout:219043332139 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 msgr2=0x7f7bc00a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 0x7f7bc00a3680 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f7bd4074000 tx=0x7f7bc4009260 comp rx=0 tx=0).stop 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 msgr2=0x7f7bd4072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 0x7f7bd4072810 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f7bd000e9f0 tx=0x7f7bd000ed00 comp rx=0 tx=0).stop 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 shutdown_connections 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7bc00a11d0 0x7f7bc00a3680 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7bd4104ea0 0x7f7bd40732d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7bd40ffe00 0x7f7bd4072810 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 --2- 172.21.15.40:0/1114340844 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7bd41007a0 0x7f7bd4072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:52.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 >> 172.21.15.40:0/1114340844 conn(0x7f7bd40fb660 msgr2=0x7f7bd40fc080 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:52.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 shutdown_connections 2023-07-14T11:10:52.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.523+0000 7f7bdb458700 1 -- 172.21.15.40:0/1114340844 wait complete. 2023-07-14T11:10:52.909 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574918 got 55834574920 for osd.1 2023-07-14T11:10:52.909 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1969012019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-07-14T11:10:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3824096162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-07-14T11:10:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1114340844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-07-14T11:10:53.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1969012019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-07-14T11:10:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3824096162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-07-14T11:10:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1114340844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-07-14T11:10:53.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1969012019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-07-14T11:10:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3824096162' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-07-14T11:10:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1114340844' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-07-14T11:10:53.068 INFO:tasks.cephadm.ceph_manager.ceph:need seq 309237645338 got 309237645341 for osd.10 2023-07-14T11:10:53.069 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:53.145 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757299 got 163208757301 for osd.5 2023-07-14T11:10:53.146 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:53.501 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182458 got 107374182461 for osd.3 2023-07-14T11:10:53.501 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:53.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 -- 172.21.15.40:0/3901252219 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 msgr2=0x7f110c072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:53.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 --2- 172.21.15.40:0/3901252219 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c072c80 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f10fc009990 tx=0x7f10fc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 -- 172.21.15.40:0/3901252219 shutdown_connections 2023-07-14T11:10:53.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 --2- 172.21.15.40:0/3901252219 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 0x7f110c10e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 --2- 172.21.15.40:0/3901252219 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c072c80 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 --2- 172.21.15.40:0/3901252219 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c10be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 -- 172.21.15.40:0/3901252219 >> 172.21.15.40:0/3901252219 conn(0x7f110c0fb670 msgr2=0x7f110c0fda80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 -- 172.21.15.40:0/3901252219 shutdown_connections 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.929+0000 7f11109d4700 1 -- 172.21.15.40:0/3901252219 wait complete. 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.930+0000 7f11109d4700 1 Processor -- start 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 -- start start 2023-07-14T11:10:53.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c0ffaf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 0x7f110c100570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f110c1a84c0 con 0x7f110c0728b0 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f110c1a8630 con 0x7f110c0731c0 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f11109d4700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f110c1a87f0 con 0x7f110c10c3d0 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f110a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c0ffaf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f1109d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:53.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f1109d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:57054/0 (socket says 172.21.15.40:57054) 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f1109d9b700 1 -- 172.21.15.40:0/838318960 learned_addr learned my addr 172.21.15.40:0/838318960 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.931+0000 7f110ad9d700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 0x7f110c100570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 -- 172.21.15.40:0/838318960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 msgr2=0x7f110c100570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 0x7f110c100570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 -- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 msgr2=0x7f110c0ffaf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c0ffaf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 -- 172.21.15.40:0/838318960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10fc0095f0 con 0x7f110c0731c0 2023-07-14T11:10:53.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f110a59c700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c0ffaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f1109d9b700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f110000e9a0 tx=0x7f110000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1100009790 con 0x7f110c0731c0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f110c1a8ad0 con 0x7f110c0731c0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1100010430 con 0x7f110c0731c0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1100018680 con 0x7f110c0731c0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.932+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f110c1a9020 con 0x7f110c0731c0 2023-07-14T11:10:53.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.933+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f110c04e210 con 0x7f110c0731c0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.935+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f11000105a0 con 0x7f110c0731c0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.937+0000 7f10fb7fe700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 0x7f10f40a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.937+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f1100014070 con 0x7f110c0731c0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.937+0000 7f110a59c700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 0x7f10f40a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.938+0000 7f110a59c700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 0x7f10f40a3600 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f10fc00b2e0 tx=0x7f10fc005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:52.941+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1100051790 con 0x7f110c0731c0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.175+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 6} v 0) v1 -- 0x7f110c1a9340 con 0x7f110c0731c0 2023-07-14T11:10:53.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.175+0000 7f10fb7fe700 1 -- 172.21.15.40:0/838318960 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 6}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f1100096210 con 0x7f110c0731c0 2023-07-14T11:10:53.561 INFO:teuthology.orchestra.run.smithi040.stdout:197568495664 2023-07-14T11:10:53.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 msgr2=0x7f10f40a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:53.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 0x7f10f40a3600 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f10fc00b2e0 tx=0x7f10fc005960 comp rx=0 tx=0).stop 2023-07-14T11:10:53.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 msgr2=0x7f110c100030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f110000e9a0 tx=0x7f110000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 shutdown_connections 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f10f40a1150 0x7f10f40a3600 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f110c10c3d0 0x7f110c100570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f110c0728b0 0x7f110c0ffaf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 --2- 172.21.15.40:0/838318960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f110c0731c0 0x7f110c100030 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 >> 172.21.15.40:0/838318960 conn(0x7f110c0fb670 msgr2=0x7f110c10e4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:53.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.178+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 shutdown_connections 2023-07-14T11:10:53.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:53.179+0000 7f11109d4700 1 -- 172.21.15.40:0/838318960 wait complete. 2023-07-14T11:10:53.760 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:54.005 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:53 smithi040 ceph-mon[97954]: pgmap v313: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:54.005 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:53 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/838318960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-07-14T11:10:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:53 smithi130 ceph-mon[129405]: pgmap v313: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:53 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/838318960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-07-14T11:10:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:53 smithi045 ceph-mon[128791]: pgmap v313: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:10:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:53 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/838318960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-07-14T11:10:54.080 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332136 got 219043332139 for osd.7 2023-07-14T11:10:54.080 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 -- 172.21.15.40:0/1714917307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd810c3e0 msgr2=0x7f3bd810e7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/1714917307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd810c3e0 0x7f3bd810e7c0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f3bd000b0a0 tx=0x7f3bd000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 -- 172.21.15.40:0/1714917307 shutdown_connections 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/1714917307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd810c3e0 0x7f3bd810e7c0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/1714917307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80728b0 0x7f3bd8072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/1714917307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd80731c0 0x7f3bd810bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 -- 172.21.15.40:0/1714917307 >> 172.21.15.40:0/1714917307 conn(0x7f3bd80fb660 msgr2=0x7f3bd80fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:54.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 -- 172.21.15.40:0/1714917307 shutdown_connections 2023-07-14T11:10:54.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.212+0000 7f3bde3cb700 1 -- 172.21.15.40:0/1714917307 wait complete. 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.213+0000 7f3bde3cb700 1 Processor -- start 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 -- start start 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 0x7f3bd8198420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd810c3e0 0x7f3bd819c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bd819cf30 con 0x7f3bd80728b0 2023-07-14T11:10:54.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bd819d0a0 con 0x7f3bd80731c0 2023-07-14T11:10:54.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bde3cb700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3bd819d260 con 0x7f3bd810c3e0 2023-07-14T11:10:54.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bd7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.214+0000 7f3bd7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42616/0 (socket says 172.21.15.40:42616) 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd7fff700 1 -- 172.21.15.40:0/3683147496 learned_addr learned my addr 172.21.15.40:0/3683147496 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 0x7f3bd8198420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bdc968700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd810c3e0 0x7f3bd819c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 -- 172.21.15.40:0/3683147496 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd810c3e0 msgr2=0x7f3bd819c890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd810c3e0 0x7f3bd819c890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 -- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 msgr2=0x7f3bd8197ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 -- 172.21.15.40:0/3683147496 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3bd0009d00 con 0x7f3bd80731c0 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd7fff700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.215+0000 7f3bd77fe700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 0x7f3bd8198420 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f3bcc00e9f0 tx=0x7f3bcc00ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:54.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.216+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bcc00c9c0 con 0x7f3bd80731c0 2023-07-14T11:10:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.216+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3bd819d540 con 0x7f3bd80731c0 2023-07-14T11:10:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.216+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3bcc010430 con 0x7f3bd80731c0 2023-07-14T11:10:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.216+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3bcc018680 con 0x7f3bd80731c0 2023-07-14T11:10:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.216+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3bd81a4d10 con 0x7f3bd80731c0 2023-07-14T11:10:54.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.217+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3bd8106d00 con 0x7f3bd80731c0 2023-07-14T11:10:54.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.219+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f3bcc010860 con 0x7f3bd80731c0 2023-07-14T11:10:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.221+0000 7f3bd57fa700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 0x7f3bc00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.221+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f3bcc014070 con 0x7f3bd80731c0 2023-07-14T11:10:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.221+0000 7f3bd7fff700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 0x7f3bc00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.222+0000 7f3bd7fff700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 0x7f3bc00a3600 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8011550 tx=0x7f3bc8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:54.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.222+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3bcc0960c0 con 0x7f3bd80731c0 2023-07-14T11:10:54.243 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738442 got 34359738444 for osd.0 2023-07-14T11:10:54.243 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:54.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.447+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f3bd804e210 con 0x7f3bd80731c0 2023-07-14T11:10:54.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.448+0000 7f3bd57fa700 1 -- 172.21.15.40:0/3683147496 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f3bcc0996e0 con 0x7f3bd80731c0 2023-07-14T11:10:54.451 INFO:teuthology.orchestra.run.smithi040.stdout:77309411396 2023-07-14T11:10:54.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.450+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 msgr2=0x7f3bc00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.450+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 0x7f3bc00a3600 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f3bc8011550 tx=0x7f3bc8009260 comp rx=0 tx=0).stop 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 msgr2=0x7f3bd8198420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 0x7f3bd8198420 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f3bcc00e9f0 tx=0x7f3bcc00ed00 comp rx=0 tx=0).stop 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 shutdown_connections 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3bc00a1150 0x7f3bc00a3600 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3bd810c3e0 0x7f3bd819c890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3bd80728b0 0x7f3bd8197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 --2- 172.21.15.40:0/3683147496 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3bd80731c0 0x7f3bd8198420 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 >> 172.21.15.40:0/3683147496 conn(0x7f3bd80fb660 msgr2=0x7f3bd8103e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:54.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 shutdown_connections 2023-07-14T11:10:54.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.451+0000 7f3bde3cb700 1 -- 172.21.15.40:0/3683147496 wait complete. 2023-07-14T11:10:54.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.992+0000 7f6d30847700 1 -- 172.21.15.40:0/1856856344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c0ffe00 msgr2=0x7f6d2c1001d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.992+0000 7f6d30847700 1 --2- 172.21.15.40:0/1856856344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c1001d0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f6d1c009990 tx=0x7f6d1c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 -- 172.21.15.40:0/1856856344 shutdown_connections 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 --2- 172.21.15.40:0/1856856344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c104ea0 0x7f6d2c10ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 --2- 172.21.15.40:0/1856856344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c1001d0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 --2- 172.21.15.40:0/1856856344 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 -- 172.21.15.40:0/1856856344 >> 172.21.15.40:0/1856856344 conn(0x7f6d2c0fb660 msgr2=0x7f6d2c0fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:54.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 -- 172.21.15.40:0/1856856344 shutdown_connections 2023-07-14T11:10:54.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.993+0000 7f6d30847700 1 -- 172.21.15.40:0/1856856344 wait complete. 2023-07-14T11:10:54.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 Processor -- start 2023-07-14T11:10:54.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 -- start start 2023-07-14T11:10:54.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c19bf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c104ea0 0x7f6d2c195ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d2c196140 con 0x7f6d2c0ffe00 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d2c1962b0 con 0x7f6d2c1007a0 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.994+0000 7f6d30847700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d2c196470 con 0x7f6d2c104ea0 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d2a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c19bf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d2a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42640/0 (socket says 172.21.15.40:42640) 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d2a59c700 1 -- 172.21.15.40:0/1645730346 learned_addr learned my addr 172.21.15.40:0/1645730346 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:54.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d2ad9d700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c104ea0 0x7f6d2c195ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 -- 172.21.15.40:0/1645730346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c104ea0 msgr2=0x7f6d2c195ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c104ea0 0x7f6d2c195ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 -- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 msgr2=0x7f6d2c19ba50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 -- 172.21.15.40:0/1645730346 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d1c0095f0 con 0x7f6d2c1007a0 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d2a59c700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:55.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.995+0000 7f6d29d9b700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c19bf90 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f6d2000e9c0 tx=0x7f6d2000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:55.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.996+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d2000c9c0 con 0x7f6d2c1007a0 2023-07-14T11:10:55.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.996+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6d20010430 con 0x7f6d2c1007a0 2023-07-14T11:10:55.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.996+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d2c196750 con 0x7f6d2c1007a0 2023-07-14T11:10:55.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.996+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d20018680 con 0x7f6d2c1007a0 2023-07-14T11:10:55.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.996+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d2c1a2a40 con 0x7f6d2c1007a0 2023-07-14T11:10:55.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.997+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d2c04e210 con 0x7f6d2c1007a0 2023-07-14T11:10:55.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:54.999+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6d2000cb20 con 0x7f6d2c1007a0 2023-07-14T11:10:55.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.000+0000 7f6d1b7fe700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 0x7f6d140a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:55.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.001+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f6d20014070 con 0x7f6d2c1007a0 2023-07-14T11:10:55.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.001+0000 7f6d2a59c700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 0x7f6d140a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:55.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.001+0000 7f6d2a59c700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 0x7f6d140a3600 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f6d1c006010 tx=0x7f6d1c00b260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:55.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.003+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6d20096400 con 0x7f6d2c1007a0 2023-07-14T11:10:55.166 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495661 got 197568495664 for osd.6 2023-07-14T11:10:55.166 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:55.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.218+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 9} v 0) v1 -- 0x7f6d2c1a2cd0 con 0x7f6d2c1007a0 2023-07-14T11:10:55.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.218+0000 7f6d1b7fe700 1 -- 172.21.15.40:0/1645730346 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 9}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f6d20099a20 con 0x7f6d2c1007a0 2023-07-14T11:10:55.221 INFO:teuthology.orchestra.run.smithi040.stdout:287762808866 2023-07-14T11:10:55.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 msgr2=0x7f6d140a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 0x7f6d140a3600 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f6d1c006010 tx=0x7f6d1c00b260 comp rx=0 tx=0).stop 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 msgr2=0x7f6d2c19bf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c19bf90 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f6d2000e9c0 tx=0x7f6d2000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 shutdown_connections 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6d140a1150 0x7f6d140a3600 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d2c104ea0 0x7f6d2c195ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d2c0ffe00 0x7f6d2c19ba50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 --2- 172.21.15.40:0/1645730346 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d2c1007a0 0x7f6d2c19bf90 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:55.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 >> 172.21.15.40:0/1645730346 conn(0x7f6d2c0fb660 msgr2=0x7f6d2c0fc160 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:55.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 shutdown_connections 2023-07-14T11:10:55.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:55.221+0000 7f6d30847700 1 -- 172.21.15.40:0/1645730346 wait complete. 2023-07-14T11:10:55.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:55 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3683147496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-07-14T11:10:55.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:55 smithi040 ceph-mon[97954]: pgmap v314: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:10:55.530 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:55 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1645730346' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2023-07-14T11:10:55.564 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411393 got 77309411396 for osd.2 2023-07-14T11:10:55.564 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:55 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3683147496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-07-14T11:10:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:55 smithi130 ceph-mon[129405]: pgmap v314: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:10:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:55 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1645730346' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2023-07-14T11:10:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:55 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3683147496' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-07-14T11:10:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:55 smithi045 ceph-mon[128791]: pgmap v314: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:10:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:55 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1645730346' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 9}]: dispatch 2023-07-14T11:10:55.951 INFO:tasks.cephadm.ceph_manager.ceph:need seq 141733920824 got 141733920826 for osd.4 2023-07-14T11:10:55.952 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:56.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 -- 172.21.15.40:0/2979238024 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f93541070e0 msgr2=0x7f935410ec60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:56.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 --2- 172.21.15.40:0/2979238024 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f93541070e0 0x7f935410ec60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f934c00b0d0 tx=0x7f934c00b3e0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 -- 172.21.15.40:0/2979238024 shutdown_connections 2023-07-14T11:10:56.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 --2- 172.21.15.40:0/2979238024 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f93541070e0 0x7f935410ec60 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 --2- 172.21.15.40:0/2979238024 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9354102040 0x7f9354102410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 --2- 172.21.15.40:0/2979238024 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 0x7f93541069b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 -- 172.21.15.40:0/2979238024 >> 172.21.15.40:0/2979238024 conn(0x7f93540fd900 msgr2=0x7f93540ffd10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:56.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 -- 172.21.15.40:0/2979238024 shutdown_connections 2023-07-14T11:10:56.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.089+0000 7f935b65d700 1 -- 172.21.15.40:0/2979238024 wait complete. 2023-07-14T11:10:56.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.091+0000 7f935b65d700 1 Processor -- start 2023-07-14T11:10:56.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.091+0000 7f935b65d700 1 -- start start 2023-07-14T11:10:56.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.091+0000 7f935b65d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 0x7f9354197f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:56.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.091+0000 7f935b65d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 0x7f9354198460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:56.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f935b65d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 0x7f935419c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:56.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f935b65d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f935419cee0 con 0x7f93541029e0 2023-07-14T11:10:56.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f935b65d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f935419d050 con 0x7f93541070e0 2023-07-14T11:10:56.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f935b65d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f935419d210 con 0x7f9354102040 2023-07-14T11:10:56.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f9359bfa700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 0x7f935419c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:56.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 0x7f9354197f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:56.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f9359bfa700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 0x7f935419c8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46598/0 (socket says 172.21.15.40:46598) 2023-07-14T11:10:56.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f9359bfa700 1 -- 172.21.15.40:0/1569760896 learned_addr learned my addr 172.21.15.40:0/1569760896 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:56.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f9358bf8700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 0x7f9354198460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 -- 172.21.15.40:0/1569760896 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 msgr2=0x7f935419c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 0x7f935419c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 -- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 msgr2=0x7f9354198460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 0x7f9354198460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.092+0000 7f93593f9700 1 -- 172.21.15.40:0/1569760896 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f934c009d00 con 0x7f9354102040 2023-07-14T11:10:56.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f93593f9700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 0x7f9354197f20 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f935000e9f0 tx=0x7f935000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:56.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f935000c9c0 con 0x7f9354102040 2023-07-14T11:10:56.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9350010430 con 0x7f9354102040 2023-07-14T11:10:56.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f935419d4f0 con 0x7f9354102040 2023-07-14T11:10:56.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9350018680 con 0x7f9354102040 2023-07-14T11:10:56.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.093+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f93541a4d50 con 0x7f9354102040 2023-07-14T11:10:56.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.096+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9338005320 con 0x7f9354102040 2023-07-14T11:10:56.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.098+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f935000cb20 con 0x7f9354102040 2023-07-14T11:10:56.101 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.099+0000 7f934a7fc700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 0x7f93400a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:56.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.099+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f9350014070 con 0x7f9354102040 2023-07-14T11:10:56.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.100+0000 7f9358bf8700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 0x7f93400a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:56.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.102+0000 7f9358bf8700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 0x7f93400a3600 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f9354199540 tx=0x7f9344009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:56.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.103+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9350051790 con 0x7f9354102040 2023-07-14T11:10:56.150 INFO:tasks.cephadm.ceph_manager.ceph:need seq 287762808864 got 287762808866 for osd.9 2023-07-14T11:10:56.151 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:56.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.336+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 8} v 0) v1 -- 0x7f9338005190 con 0x7f9354102040 2023-07-14T11:10:56.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.337+0000 7f934a7fc700 1 -- 172.21.15.40:0/1569760896 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 8}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f93500cee60 con 0x7f9354102040 2023-07-14T11:10:56.340 INFO:teuthology.orchestra.run.smithi040.stdout:253403070503 2023-07-14T11:10:56.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.340+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 msgr2=0x7f93400a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.340+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 0x7f93400a3600 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f9354199540 tx=0x7f9344009260 comp rx=0 tx=0).stop 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.340+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 msgr2=0x7f9354197f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.340+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 0x7f9354197f20 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f935000e9f0 tx=0x7f935000ed00 comp rx=0 tx=0).stop 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 shutdown_connections 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f93400a1150 0x7f93400a3600 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f93541070e0 0x7f935419c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9354102040 0x7f9354197f20 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 --2- 172.21.15.40:0/1569760896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93541029e0 0x7f9354198460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:56.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 >> 172.21.15.40:0/1569760896 conn(0x7f93540fd900 msgr2=0x7f93540fe2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:56.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 shutdown_connections 2023-07-14T11:10:56.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:56.342+0000 7f935b65d700 1 -- 172.21.15.40:0/1569760896 wait complete. 2023-07-14T11:10:56.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:56 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1569760896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2023-07-14T11:10:56.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:56 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1569760896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2023-07-14T11:10:56.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:56 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1569760896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 8}]: dispatch 2023-07-14T11:10:56.934 INFO:tasks.cephadm.ceph_manager.ceph:need seq 253403070501 got 253403070503 for osd.8 2023-07-14T11:10:56.935 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:57.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.562+0000 7f2c45501700 1 -- 172.21.15.40:0/1279542554 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c40104ea0 msgr2=0x7f2c4010ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.562+0000 7f2c45501700 1 --2- 172.21.15.40:0/1279542554 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c40104ea0 0x7f2c4010ca20 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f2c38009fc0 tx=0x7f2c3800b330 comp rx=0 tx=0).stop 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 -- 172.21.15.40:0/1279542554 shutdown_connections 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 --2- 172.21.15.40:0/1279542554 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c40104ea0 0x7f2c4010ca20 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 --2- 172.21.15.40:0/1279542554 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c400ffe00 0x7f2c401001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 --2- 172.21.15.40:0/1279542554 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c401007a0 0x7f2c40104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 -- 172.21.15.40:0/1279542554 >> 172.21.15.40:0/1279542554 conn(0x7f2c400fb660 msgr2=0x7f2c400fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:57.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 -- 172.21.15.40:0/1279542554 shutdown_connections 2023-07-14T11:10:57.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.563+0000 7f2c45501700 1 -- 172.21.15.40:0/1279542554 wait complete. 2023-07-14T11:10:57.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.564+0000 7f2c45501700 1 Processor -- start 2023-07-14T11:10:57.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.564+0000 7f2c45501700 1 -- start start 2023-07-14T11:10:57.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:57.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 0x7f2c401962a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 0x7f2c4019a710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c4019ad20 con 0x7f2c401007a0 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c4019ae90 con 0x7f2c40104ea0 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c45501700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c4019b050 con 0x7f2c400ffe00 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c3effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c3e7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 0x7f2c401962a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:57.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c3effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50882/0 (socket says 172.21.15.40:50882) 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c3effd700 1 -- 172.21.15.40:0/4233593975 learned_addr learned my addr 172.21.15.40:0/4233593975 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.565+0000 7f2c3f7fe700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 0x7f2c4019a710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 -- 172.21.15.40:0/4233593975 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 msgr2=0x7f2c40195d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 -- 172.21.15.40:0/4233593975 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 msgr2=0x7f2c4019a710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 0x7f2c4019a710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 -- 172.21.15.40:0/4233593975 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2c300095f0 con 0x7f2c401007a0 2023-07-14T11:10:57.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3effd700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:57.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3f7fe700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 0x7f2c4019a710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:10:57.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c3e7fc700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 0x7f2c401962a0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f2c3400e9c0 tx=0x7f2c3400ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:57.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c3400cb70 con 0x7f2c401007a0 2023-07-14T11:10:57.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2c34009520 con 0x7f2c401007a0 2023-07-14T11:10:57.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2c3400ee60 con 0x7f2c401007a0 2023-07-14T11:10:57.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.566+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2c34003c60 con 0x7f2c401007a0 2023-07-14T11:10:57.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.567+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2c38009c40 con 0x7f2c401007a0 2023-07-14T11:10:57.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.568+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2c4004e210 con 0x7f2c401007a0 2023-07-14T11:10:57.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.570+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f2c34003dc0 con 0x7f2c401007a0 2023-07-14T11:10:57.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.571+0000 7f2c27fff700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 0x7f2c280a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:10:57.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.571+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f2c34014070 con 0x7f2c401007a0 2023-07-14T11:10:57.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.571+0000 7f2c3effd700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 0x7f2c280a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:10:57.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.572+0000 7f2c3effd700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 0x7f2c280a3600 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f2c30005d10 tx=0x7f2c30009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:10:57.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.573+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f2c34050790 con 0x7f2c401007a0 2023-07-14T11:10:57.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.788+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 11} v 0) v1 -- 0x7f2c401970c0 con 0x7f2c401007a0 2023-07-14T11:10:57.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.788+0000 7f2c27fff700 1 -- 172.21.15.40:0/4233593975 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 11}]=0 v0) v1 ==== 75+0+13 (secure 0 0 0) 0x7f2c34096400 con 0x7f2c401007a0 2023-07-14T11:10:57.791 INFO:teuthology.orchestra.run.smithi040.stdout:339302416412 2023-07-14T11:10:57.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:57 smithi130 ceph-mon[129405]: pgmap v315: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:10:57.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 msgr2=0x7f2c280a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:57.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 0x7f2c280a3600 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f2c30005d10 tx=0x7f2c30009310 comp rx=0 tx=0).stop 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 msgr2=0x7f2c401962a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 0x7f2c401962a0 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f2c3400e9c0 tx=0x7f2c3400ecd0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 shutdown_connections 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2c280a1150 0x7f2c280a3600 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2c40104ea0 0x7f2c4019a710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2c400ffe00 0x7f2c40195d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 --2- 172.21.15.40:0/4233593975 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2c401007a0 0x7f2c401962a0 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:10:57.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 >> 172.21.15.40:0/4233593975 conn(0x7f2c400fb660 msgr2=0x7f2c400fc080 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:10:57.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 shutdown_connections 2023-07-14T11:10:57.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:10:57.791+0000 7f2c45501700 1 -- 172.21.15.40:0/4233593975 wait complete. 2023-07-14T11:10:57.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:57 smithi040 ceph-mon[97954]: pgmap v315: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:10:57.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:57 smithi045 ceph-mon[128791]: pgmap v315: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:10:58.199 INFO:tasks.cephadm.ceph_manager.ceph:need seq 339302416409 got 339302416412 for osd.11 2023-07-14T11:10:58.199 DEBUG:teuthology.parallel:result is None 2023-07-14T11:10:58.200 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-07-14T11:10:58.200 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph pg dump --format=json 2023-07-14T11:10:58.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4233593975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2023-07-14T11:10:58.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4233593975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2023-07-14T11:10:58.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4233593975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 11}]: dispatch 2023-07-14T11:10:58.953 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:10:59.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:10:59 smithi040 ceph-mon[97954]: pgmap v316: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:10:59.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:10:59 smithi130 ceph-mon[129405]: pgmap v316: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:10:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:10:59 smithi045 ceph-mon[128791]: pgmap v316: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:00.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.488+0000 7f8b59932700 1 -- 172.21.15.40:0/3185169679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b54101700 msgr2=0x7f8b54109c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 --2- 172.21.15.40:0/3185169679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b54101700 0x7f8b54109c00 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f8b480098f0 tx=0x7f8b48009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 -- 172.21.15.40:0/3185169679 shutdown_connections 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 --2- 172.21.15.40:0/3185169679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b5410a140 0x7f8b5410c580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 --2- 172.21.15.40:0/3185169679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54100df0 0x7f8b541011c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 --2- 172.21.15.40:0/3185169679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b54101700 0x7f8b54109c00 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 -- 172.21.15.40:0/3185169679 >> 172.21.15.40:0/3185169679 conn(0x7f8b54075b70 msgr2=0x7f8b54075f70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:00.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 -- 172.21.15.40:0/3185169679 shutdown_connections 2023-07-14T11:11:00.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.489+0000 7f8b59932700 1 -- 172.21.15.40:0/3185169679 wait complete. 2023-07-14T11:11:00.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 Processor -- start 2023-07-14T11:11:00.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 -- start start 2023-07-14T11:11:00.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:00.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54101700 0x7f8b541983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b5410a140 0x7f8b5419c850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b5419ce60 con 0x7f8b5410a140 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b5419cfd0 con 0x7f8b54101700 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.490+0000 7f8b59932700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b5419d190 con 0x7f8b54100df0 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50910/0 (socket says 172.21.15.40:50910) 2023-07-14T11:11:00.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 -- 172.21.15.40:0/3037199121 learned_addr learned my addr 172.21.15.40:0/3037199121 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b537fe700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b5410a140 0x7f8b5419c850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b527fc700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54101700 0x7f8b541983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 -- 172.21.15.40:0/3037199121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54101700 msgr2=0x7f8b541983e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54101700 0x7f8b541983e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 -- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b5410a140 msgr2=0x7f8b5419c850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b5410a140 0x7f8b5419c850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 -- 172.21.15.40:0/3037199121 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b48009520 con 0x7f8b54100df0 2023-07-14T11:11:00.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.491+0000 7f8b52ffd700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f8b4400e9c0 tx=0x7f8b4400ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:00.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.492+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b44004a30 con 0x7f8b54100df0 2023-07-14T11:11:00.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.492+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8b4400cc80 con 0x7f8b54100df0 2023-07-14T11:11:00.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.492+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b5419d470 con 0x7f8b54100df0 2023-07-14T11:11:00.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.492+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8b440054e0 con 0x7f8b54100df0 2023-07-14T11:11:00.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.492+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b541a4cd0 con 0x7f8b54100df0 2023-07-14T11:11:00.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.493+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b541076c0 con 0x7f8b54100df0 2023-07-14T11:11:00.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.495+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8b44004b90 con 0x7f8b54100df0 2023-07-14T11:11:00.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.496+0000 7f8b58930700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 0x7f8b3c0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:00.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.496+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f8b44014070 con 0x7f8b54100df0 2023-07-14T11:11:00.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.497+0000 7f8b527fc700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 0x7f8b3c0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:00.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.497+0000 7f8b527fc700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 0x7f8b3c0a3680 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f8b48005e90 tx=0x7f8b48019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:00.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.499+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8b440966e0 con 0x7f8b54100df0 2023-07-14T11:11:00.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:00 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:00 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:00 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:11:00.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:00 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:11:00.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:00 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.701+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f8b5402c8c0 con 0x7f8b3c0a11d0 2023-07-14T11:11:00.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.716+0000 7f8b58930700 1 -- 172.21.15.40:0/3037199121 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+426786 (secure 0 0 0) 0x7f8b5402c8c0 con 0x7f8b3c0a11d0 2023-07-14T11:11:00.723 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:00.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 msgr2=0x7f8b3c0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:00.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 0x7f8b3c0a3680 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f8b48005e90 tx=0x7f8b48019040 comp rx=0 tx=0).stop 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 msgr2=0x7f8b54197ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f8b4400e9c0 tx=0x7f8b4400ecd0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 shutdown_connections 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8b3c0a11d0 0x7f8b3c0a3680 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b5410a140 0x7f8b5419c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8b54100df0 0x7f8b54197ea0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 --2- 172.21.15.40:0/3037199121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8b54101700 0x7f8b541983e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:00.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.721+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 >> 172.21.15.40:0/3037199121 conn(0x7f8b54075b70 msgr2=0x7f8b540fefb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:00.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.722+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 shutdown_connections 2023-07-14T11:11:00.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:00.722+0000 7f8b59932700 1 -- 172.21.15.40:0/3037199121 wait complete. 2023-07-14T11:11:00.727 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:11:00.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:00 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:00 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:00 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:11:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:00 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:11:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:00 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:00 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:00 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:00 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:11:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:00 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:11:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:00 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' 2023-07-14T11:11:01.260 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":317,"stamp":"2023-07-14T11:11:00.644878+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":600392,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":597,"num_read_kb":371,"num_write":390,"num_write_kb":3005,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":320,"ondisk_log_size":320,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337236,"kb_used_data":13812,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604588,"statfs":{"total":1151940427776,"available":1151595098112,"internally_reserved":0,"allocated":14143488,"data_stored":7664382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":130,"num_read_kb":70,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003371"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738359+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:10:07.738359+0000","last_peered":"2023-07-14T11:10:07.738359+0000","last_clean":"2023-07-14T11:10:07.738359+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:10:07.738359+0000","last_undegraded":"2023-07-14T11:10:07.738359+0000","last_fullsized":"2023-07-14T11:10:07.738359+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:21:10.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141620+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:10:08.141620+0000","last_peered":"2023-07-14T11:10:08.141620+0000","last_clean":"2023-07-14T11:10:08.141620+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:10:08.141620+0000","last_undegraded":"2023-07-14T11:10:08.141620+0000","last_fullsized":"2023-07-14T11:10:08.141620+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:43:13.354815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142202+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:10:08.142202+0000","last_peered":"2023-07-14T11:10:08.142202+0000","last_clean":"2023-07-14T11:10:08.142202+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:10:08.142202+0000","last_undegraded":"2023-07-14T11:10:08.142202+0000","last_fullsized":"2023-07-14T11:10:08.142202+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:25:21.721235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823979+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:10:07.823979+0000","last_peered":"2023-07-14T11:10:07.823979+0000","last_clean":"2023-07-14T11:10:07.823979+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:10:07.823979+0000","last_undegraded":"2023-07-14T11:10:07.823979+0000","last_fullsized":"2023-07-14T11:10:07.823979+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:50:39.156238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734553+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:10:07.734553+0000","last_peered":"2023-07-14T11:10:07.734553+0000","last_clean":"2023-07-14T11:10:07.734553+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:10:07.734553+0000","last_undegraded":"2023-07-14T11:10:07.734553+0000","last_fullsized":"2023-07-14T11:10:07.734553+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:08:20.010349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142872+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:10:08.142872+0000","last_peered":"2023-07-14T11:10:08.142872+0000","last_clean":"2023-07-14T11:10:08.142872+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:10:08.142872+0000","last_undegraded":"2023-07-14T11:10:08.142872+0000","last_fullsized":"2023-07-14T11:10:08.142872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:38:58.846023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736546+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:10:07.736546+0000","last_peered":"2023-07-14T11:10:07.736546+0000","last_clean":"2023-07-14T11:10:07.736546+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:10:07.736546+0000","last_undegraded":"2023-07-14T11:10:07.736546+0000","last_fullsized":"2023-07-14T11:10:07.736546+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:29:36.124497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739054+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:10:07.739054+0000","last_peered":"2023-07-14T11:10:07.739054+0000","last_clean":"2023-07-14T11:10:07.739054+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:10:07.739054+0000","last_undegraded":"2023-07-14T11:10:07.739054+0000","last_fullsized":"2023-07-14T11:10:07.739054+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:48:52.238878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735166+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:10:07.735166+0000","last_peered":"2023-07-14T11:10:07.735166+0000","last_clean":"2023-07-14T11:10:07.735166+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:10:07.735166+0000","last_undegraded":"2023-07-14T11:10:07.735166+0000","last_fullsized":"2023-07-14T11:10:07.735166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:45.770708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143166+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:10:08.143166+0000","last_peered":"2023-07-14T11:10:08.143166+0000","last_clean":"2023-07-14T11:10:08.143166+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:10:08.143166+0000","last_undegraded":"2023-07-14T11:10:08.143166+0000","last_fullsized":"2023-07-14T11:10:08.143166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:11.980357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737468+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:10:07.737468+0000","last_peered":"2023-07-14T11:10:07.737468+0000","last_clean":"2023-07-14T11:10:07.737468+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:10:07.737468+0000","last_undegraded":"2023-07-14T11:10:07.737468+0000","last_fullsized":"2023-07-14T11:10:07.737468+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:31:39.003266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823717+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:10:07.823717+0000","last_peered":"2023-07-14T11:10:07.823717+0000","last_clean":"2023-07-14T11:10:07.823717+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:10:07.823717+0000","last_undegraded":"2023-07-14T11:10:07.823717+0000","last_fullsized":"2023-07-14T11:10:07.823717+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:03:29.844122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734676+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:10:07.734676+0000","last_peered":"2023-07-14T11:10:07.734676+0000","last_clean":"2023-07-14T11:10:07.734676+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:10:07.734676+0000","last_undegraded":"2023-07-14T11:10:07.734676+0000","last_fullsized":"2023-07-14T11:10:07.734676+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:10:43.308729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141270+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:10:08.141270+0000","last_peered":"2023-07-14T11:10:08.141270+0000","last_clean":"2023-07-14T11:10:08.141270+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:10:08.141270+0000","last_undegraded":"2023-07-14T11:10:08.141270+0000","last_fullsized":"2023-07-14T11:10:08.141270+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:20:00.054130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141301+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:10:08.141301+0000","last_peered":"2023-07-14T11:10:08.141301+0000","last_clean":"2023-07-14T11:10:08.141301+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:10:08.141301+0000","last_undegraded":"2023-07-14T11:10:08.141301+0000","last_fullsized":"2023-07-14T11:10:08.141301+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:38:22.587911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734282+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:10:07.734282+0000","last_peered":"2023-07-14T11:10:07.734282+0000","last_clean":"2023-07-14T11:10:07.734282+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:10:07.734282+0000","last_undegraded":"2023-07-14T11:10:07.734282+0000","last_fullsized":"2023-07-14T11:10:07.734282+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:53:57.650717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738135+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:10:07.738135+0000","last_peered":"2023-07-14T11:10:07.738135+0000","last_clean":"2023-07-14T11:10:07.738135+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:10:07.738135+0000","last_undegraded":"2023-07-14T11:10:07.738135+0000","last_fullsized":"2023-07-14T11:10:07.738135+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:48:17.654653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740775+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:10:07.740775+0000","last_peered":"2023-07-14T11:10:07.740775+0000","last_clean":"2023-07-14T11:10:07.740775+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:10:07.740775+0000","last_undegraded":"2023-07-14T11:10:07.740775+0000","last_fullsized":"2023-07-14T11:10:07.740775+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:36:51.637622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737923+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:10:07.737923+0000","last_peered":"2023-07-14T11:10:07.737923+0000","last_clean":"2023-07-14T11:10:07.737923+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:07.737923+0000","last_undegraded":"2023-07-14T11:10:07.737923+0000","last_fullsized":"2023-07-14T11:10:07.737923+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:45.243063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738752+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:10:07.738752+0000","last_peered":"2023-07-14T11:10:07.738752+0000","last_clean":"2023-07-14T11:10:07.738752+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:10:07.738752+0000","last_undegraded":"2023-07-14T11:10:07.738752+0000","last_fullsized":"2023-07-14T11:10:07.738752+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:12:05.905433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"90'13","reported_seq":80,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:55.458551+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:10:55.458551+0000","last_peered":"2023-07-14T11:10:55.458551+0000","last_clean":"2023-07-14T11:10:55.458551+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:10:55.458551+0000","last_undegraded":"2023-07-14T11:10:55.458551+0000","last_fullsized":"2023-07-14T11:10:55.458551+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:21:25.983167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":41,"num_read_kb":41,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:53.251986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822942+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:10:07.822942+0000","last_peered":"2023-07-14T11:10:07.822942+0000","last_clean":"2023-07-14T11:10:07.822942+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:10:07.822942+0000","last_undegraded":"2023-07-14T11:10:07.822942+0000","last_fullsized":"2023-07-14T11:10:07.822942+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:32:00.530574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"90'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:56.375572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142002+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:10:08.142002+0000","last_peered":"2023-07-14T11:10:08.142002+0000","last_clean":"2023-07-14T11:10:08.142002+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:10:08.142002+0000","last_undegraded":"2023-07-14T11:10:08.142002+0000","last_fullsized":"2023-07-14T11:10:08.142002+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:26.963699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823419+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:10:07.823419+0000","last_peered":"2023-07-14T11:10:07.823419+0000","last_clean":"2023-07-14T11:10:07.823419+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:10:07.823419+0000","last_undegraded":"2023-07-14T11:10:07.823419+0000","last_fullsized":"2023-07-14T11:10:07.823419+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:43.101537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'12","reported_seq":39,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:17.382526+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:10:17.382526+0000","last_peered":"2023-07-14T11:10:17.382526+0000","last_clean":"2023-07-14T11:10:17.382526+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:10:17.382526+0000","last_undegraded":"2023-07-14T11:10:17.382526+0000","last_fullsized":"2023-07-14T11:10:17.382526+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:26:07.506765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734736+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:10:07.734736+0000","last_peered":"2023-07-14T11:10:07.734736+0000","last_clean":"2023-07-14T11:10:07.734736+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:10:07.734736+0000","last_undegraded":"2023-07-14T11:10:07.734736+0000","last_fullsized":"2023-07-14T11:10:07.734736+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:10:30.512188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822955+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:10:07.822955+0000","last_peered":"2023-07-14T11:10:07.822955+0000","last_clean":"2023-07-14T11:10:07.822955+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:10:07.822955+0000","last_undegraded":"2023-07-14T11:10:07.822955+0000","last_fullsized":"2023-07-14T11:10:07.822955+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:44.974006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376665+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:10:16.376665+0000","last_peered":"2023-07-14T11:10:16.376665+0000","last_clean":"2023-07-14T11:10:16.376665+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:10:16.376665+0000","last_undegraded":"2023-07-14T11:10:16.376665+0000","last_fullsized":"2023-07-14T11:10:16.376665+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:41:26.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740246+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:10:07.740246+0000","last_peered":"2023-07-14T11:10:07.740246+0000","last_clean":"2023-07-14T11:10:07.740246+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:10:07.740246+0000","last_undegraded":"2023-07-14T11:10:07.740246+0000","last_fullsized":"2023-07-14T11:10:07.740246+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:32.963491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738142+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:10:07.738142+0000","last_peered":"2023-07-14T11:10:07.738142+0000","last_clean":"2023-07-14T11:10:07.738142+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:10:07.738142+0000","last_undegraded":"2023-07-14T11:10:07.738142+0000","last_fullsized":"2023-07-14T11:10:07.738142+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:51:58.731259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"90'5","reported_seq":76,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:55.418246+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:10:55.418246+0000","last_peered":"2023-07-14T11:10:55.418246+0000","last_clean":"2023-07-14T11:10:55.418246+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:10:55.418246+0000","last_undegraded":"2023-07-14T11:10:55.418246+0000","last_fullsized":"2023-07-14T11:10:55.418246+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:06:48.309845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":45,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734873+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:10:07.734873+0000","last_peered":"2023-07-14T11:10:07.734873+0000","last_clean":"2023-07-14T11:10:07.734873+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:10:07.734873+0000","last_undegraded":"2023-07-14T11:10:07.734873+0000","last_fullsized":"2023-07-14T11:10:07.734873+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:48.090320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738348+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:10:07.738348+0000","last_peered":"2023-07-14T11:10:07.738348+0000","last_clean":"2023-07-14T11:10:07.738348+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:10:07.738348+0000","last_undegraded":"2023-07-14T11:10:07.738348+0000","last_fullsized":"2023-07-14T11:10:07.738348+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:36:58.104207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369266+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:10:15.369266+0000","last_peered":"2023-07-14T11:10:15.369266+0000","last_clean":"2023-07-14T11:10:15.369266+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:10:15.369266+0000","last_undegraded":"2023-07-14T11:10:15.369266+0000","last_fullsized":"2023-07-14T11:10:15.369266+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:38:46.335485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.745285+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:10:07.745285+0000","last_peered":"2023-07-14T11:10:07.745285+0000","last_clean":"2023-07-14T11:10:07.745285+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:10:07.745285+0000","last_undegraded":"2023-07-14T11:10:07.745285+0000","last_fullsized":"2023-07-14T11:10:07.745285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:17.289951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737973+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:10:07.737973+0000","last_peered":"2023-07-14T11:10:07.737973+0000","last_clean":"2023-07-14T11:10:07.737973+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:10:07.737973+0000","last_undegraded":"2023-07-14T11:10:07.737973+0000","last_fullsized":"2023-07-14T11:10:07.737973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:07:16.888150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.360843+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:10:14.360843+0000","last_peered":"2023-07-14T11:10:14.360843+0000","last_clean":"2023-07-14T11:10:14.360843+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:10:14.360843+0000","last_undegraded":"2023-07-14T11:10:14.360843+0000","last_fullsized":"2023-07-14T11:10:14.360843+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:53:35.630621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738872+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:10:07.738872+0000","last_peered":"2023-07-14T11:10:07.738872+0000","last_clean":"2023-07-14T11:10:07.738872+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:10:07.738872+0000","last_undegraded":"2023-07-14T11:10:07.738872+0000","last_fullsized":"2023-07-14T11:10:07.738872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:01:08.293055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141456+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:10:08.141456+0000","last_peered":"2023-07-14T11:10:08.141456+0000","last_clean":"2023-07-14T11:10:08.141456+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:10:08.141456+0000","last_undegraded":"2023-07-14T11:10:08.141456+0000","last_fullsized":"2023-07-14T11:10:08.141456+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:14:59.191235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375874+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:10:16.375874+0000","last_peered":"2023-07-14T11:10:16.375874+0000","last_clean":"2023-07-14T11:10:16.375874+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:10:16.375874+0000","last_undegraded":"2023-07-14T11:10:16.375874+0000","last_fullsized":"2023-07-14T11:10:16.375874+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:39:15.619767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738091+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:10:07.738091+0000","last_peered":"2023-07-14T11:10:07.738091+0000","last_clean":"2023-07-14T11:10:07.738091+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:10:07.738091+0000","last_undegraded":"2023-07-14T11:10:07.738091+0000","last_fullsized":"2023-07-14T11:10:07.738091+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:48:41.840080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739132+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:10:07.739132+0000","last_peered":"2023-07-14T11:10:07.739132+0000","last_clean":"2023-07-14T11:10:07.739132+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:10:07.739132+0000","last_undegraded":"2023-07-14T11:10:07.739132+0000","last_fullsized":"2023-07-14T11:10:07.739132+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:39:26.932728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'4","reported_seq":119,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:59.424262+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:10:59.424262+0000","last_peered":"2023-07-14T11:10:59.424262+0000","last_clean":"2023-07-14T11:10:59.424262+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:10:59.424262+0000","last_undegraded":"2023-07-14T11:10:59.424262+0000","last_fullsized":"2023-07-14T11:10:59.424262+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:10:05.049869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":89,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735216+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:10:07.735216+0000","last_peered":"2023-07-14T11:10:07.735216+0000","last_clean":"2023-07-14T11:10:07.735216+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:10:07.735216+0000","last_undegraded":"2023-07-14T11:10:07.735216+0000","last_fullsized":"2023-07-14T11:10:07.735216+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:50.720818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737634+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:10:07.737634+0000","last_peered":"2023-07-14T11:10:07.737634+0000","last_clean":"2023-07-14T11:10:07.737634+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:10:07.737634+0000","last_undegraded":"2023-07-14T11:10:07.737634+0000","last_fullsized":"2023-07-14T11:10:07.737634+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:25:31.401998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'4","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375310+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:10:16.375310+0000","last_peered":"2023-07-14T11:10:16.375310+0000","last_clean":"2023-07-14T11:10:16.375310+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:10:16.375310+0000","last_undegraded":"2023-07-14T11:10:16.375310+0000","last_fullsized":"2023-07-14T11:10:16.375310+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:10:40.132095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142089+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:10:08.142089+0000","last_peered":"2023-07-14T11:10:08.142089+0000","last_clean":"2023-07-14T11:10:08.142089+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:10:08.142089+0000","last_undegraded":"2023-07-14T11:10:08.142089+0000","last_fullsized":"2023-07-14T11:10:08.142089+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:56:07.808822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823945+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:10:07.823945+0000","last_peered":"2023-07-14T11:10:07.823945+0000","last_clean":"2023-07-14T11:10:07.823945+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:10:07.823945+0000","last_undegraded":"2023-07-14T11:10:07.823945+0000","last_fullsized":"2023-07-14T11:10:07.823945+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:16:13.828703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.351487+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:10:14.351487+0000","last_peered":"2023-07-14T11:10:14.351487+0000","last_clean":"2023-07-14T11:10:14.351487+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:10:14.351487+0000","last_undegraded":"2023-07-14T11:10:14.351487+0000","last_fullsized":"2023-07-14T11:10:14.351487+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:30:17.168629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738962+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:10:07.738962+0000","last_peered":"2023-07-14T11:10:07.738962+0000","last_clean":"2023-07-14T11:10:07.738962+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:10:07.738962+0000","last_undegraded":"2023-07-14T11:10:07.738962+0000","last_fullsized":"2023-07-14T11:10:07.738962+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:25:22.187958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737948+0000","last_change":"2023-07-14T11:09:47.624060+0000","last_active":"2023-07-14T11:10:07.737948+0000","last_peered":"2023-07-14T11:10:07.737948+0000","last_clean":"2023-07-14T11:10:07.737948+0000","last_became_active":"2023-07-14T11:09:47.623605+0000","last_became_peered":"2023-07-14T11:09:47.623605+0000","last_unstale":"2023-07-14T11:10:07.737948+0000","last_undegraded":"2023-07-14T11:10:07.737948+0000","last_fullsized":"2023-07-14T11:10:07.737948+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:17:15.881693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"90'14","reported_seq":134,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:59.460658+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:10:59.460658+0000","last_peered":"2023-07-14T11:10:59.460658+0000","last_clean":"2023-07-14T11:10:59.460658+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:10:59.460658+0000","last_undegraded":"2023-07-14T11:10:59.460658+0000","last_fullsized":"2023-07-14T11:10:59.460658+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:01:52.717646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3794,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":94,"num_read_kb":51,"num_write":16,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734684+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:10:07.734684+0000","last_peered":"2023-07-14T11:10:07.734684+0000","last_clean":"2023-07-14T11:10:07.734684+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:10:07.734684+0000","last_undegraded":"2023-07-14T11:10:07.734684+0000","last_fullsized":"2023-07-14T11:10:07.734684+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:55.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740741+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:10:07.740741+0000","last_peered":"2023-07-14T11:10:07.740741+0000","last_clean":"2023-07-14T11:10:07.740741+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:10:07.740741+0000","last_undegraded":"2023-07-14T11:10:07.740741+0000","last_fullsized":"2023-07-14T11:10:07.740741+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:37.113063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"90'9","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376065+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:10:16.376065+0000","last_peered":"2023-07-14T11:10:16.376065+0000","last_clean":"2023-07-14T11:10:16.376065+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:10:16.376065+0000","last_undegraded":"2023-07-14T11:10:16.376065+0000","last_fullsized":"2023-07-14T11:10:16.376065+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:39:38.915458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734456+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:10:07.734456+0000","last_peered":"2023-07-14T11:10:07.734456+0000","last_clean":"2023-07-14T11:10:07.734456+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:10:07.734456+0000","last_undegraded":"2023-07-14T11:10:07.734456+0000","last_fullsized":"2023-07-14T11:10:07.734456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:25:56.604869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744522+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:10:07.744522+0000","last_peered":"2023-07-14T11:10:07.744522+0000","last_clean":"2023-07-14T11:10:07.744522+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:10:07.744522+0000","last_undegraded":"2023-07-14T11:10:07.744522+0000","last_fullsized":"2023-07-14T11:10:07.744522+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:08.719446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375209+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:10:16.375209+0000","last_peered":"2023-07-14T11:10:16.375209+0000","last_clean":"2023-07-14T11:10:16.375209+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:10:16.375209+0000","last_undegraded":"2023-07-14T11:10:16.375209+0000","last_fullsized":"2023-07-14T11:10:16.375209+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:12:45.811503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736758+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:10:07.736758+0000","last_peered":"2023-07-14T11:10:07.736758+0000","last_clean":"2023-07-14T11:10:07.736758+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:10:07.736758+0000","last_undegraded":"2023-07-14T11:10:07.736758+0000","last_fullsized":"2023-07-14T11:10:07.736758+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:28.882720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740995+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:10:07.740995+0000","last_peered":"2023-07-14T11:10:07.740995+0000","last_clean":"2023-07-14T11:10:07.740995+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:10:07.740995+0000","last_undegraded":"2023-07-14T11:10:07.740995+0000","last_fullsized":"2023-07-14T11:10:07.740995+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:35.770995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":73,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:55.440529+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:10:55.440529+0000","last_peered":"2023-07-14T11:10:55.440529+0000","last_clean":"2023-07-14T11:10:55.440529+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:10:55.440529+0000","last_undegraded":"2023-07-14T11:10:55.440529+0000","last_fullsized":"2023-07-14T11:10:55.440529+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:57:32.688631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":43,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737387+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:10:07.737387+0000","last_peered":"2023-07-14T11:10:07.737387+0000","last_clean":"2023-07-14T11:10:07.737387+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:10:07.737387+0000","last_undegraded":"2023-07-14T11:10:07.737387+0000","last_fullsized":"2023-07-14T11:10:07.737387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:18:06.626041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734847+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:10:07.734847+0000","last_peered":"2023-07-14T11:10:07.734847+0000","last_clean":"2023-07-14T11:10:07.734847+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:10:07.734847+0000","last_undegraded":"2023-07-14T11:10:07.734847+0000","last_fullsized":"2023-07-14T11:10:07.734847+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:14:05.500061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":44,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375086+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:10:16.375086+0000","last_peered":"2023-07-14T11:10:16.375086+0000","last_clean":"2023-07-14T11:10:16.375086+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:10:16.375086+0000","last_undegraded":"2023-07-14T11:10:16.375086+0000","last_fullsized":"2023-07-14T11:10:16.375086+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:26:26.993318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741882+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:10:07.741882+0000","last_peered":"2023-07-14T11:10:07.741882+0000","last_clean":"2023-07-14T11:10:07.741882+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:10:07.741882+0000","last_undegraded":"2023-07-14T11:10:07.741882+0000","last_fullsized":"2023-07-14T11:10:07.741882+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:38:12.515688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741815+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:10:07.741815+0000","last_peered":"2023-07-14T11:10:07.741815+0000","last_clean":"2023-07-14T11:10:07.741815+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:10:07.741815+0000","last_undegraded":"2023-07-14T11:10:07.741815+0000","last_fullsized":"2023-07-14T11:10:07.741815+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:39.325247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"90'2","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.349371+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:10:14.349371+0000","last_peered":"2023-07-14T11:10:14.349371+0000","last_clean":"2023-07-14T11:10:14.349371+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:10:14.349371+0000","last_undegraded":"2023-07-14T11:10:14.349371+0000","last_fullsized":"2023-07-14T11:10:14.349371+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:16.215502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738122+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:10:07.738122+0000","last_peered":"2023-07-14T11:10:07.738122+0000","last_clean":"2023-07-14T11:10:07.738122+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:10:07.738122+0000","last_undegraded":"2023-07-14T11:10:07.738122+0000","last_fullsized":"2023-07-14T11:10:07.738122+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:28.226381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734949+0000","last_change":"2023-07-14T11:09:47.736950+0000","last_active":"2023-07-14T11:10:07.734949+0000","last_peered":"2023-07-14T11:10:07.734949+0000","last_clean":"2023-07-14T11:10:07.734949+0000","last_became_active":"2023-07-14T11:09:47.736506+0000","last_became_peered":"2023-07-14T11:09:47.736506+0000","last_unstale":"2023-07-14T11:10:07.734949+0000","last_undegraded":"2023-07-14T11:10:07.734949+0000","last_fullsized":"2023-07-14T11:10:07.734949+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:24.567987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:11:19.303557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142811+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:10:08.142811+0000","last_peered":"2023-07-14T11:10:08.142811+0000","last_clean":"2023-07-14T11:10:08.142811+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:08.142811+0000","last_undegraded":"2023-07-14T11:10:08.142811+0000","last_fullsized":"2023-07-14T11:10:08.142811+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:51:22.856363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735220+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:10:07.735220+0000","last_peered":"2023-07-14T11:10:07.735220+0000","last_clean":"2023-07-14T11:10:07.735220+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:10:07.735220+0000","last_undegraded":"2023-07-14T11:10:07.735220+0000","last_fullsized":"2023-07-14T11:10:07.735220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:52:35.457076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'5","reported_seq":75,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:58.464547+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:10:58.464547+0000","last_peered":"2023-07-14T11:10:58.464547+0000","last_clean":"2023-07-14T11:10:58.464547+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:10:58.464547+0000","last_undegraded":"2023-07-14T11:10:58.464547+0000","last_fullsized":"2023-07-14T11:10:58.464547+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:49:44.032115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143050+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:10:08.143050+0000","last_peered":"2023-07-14T11:10:08.143050+0000","last_clean":"2023-07-14T11:10:08.143050+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:10:08.143050+0000","last_undegraded":"2023-07-14T11:10:08.143050+0000","last_fullsized":"2023-07-14T11:10:08.143050+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:04:11.876243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734819+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:10:07.734819+0000","last_peered":"2023-07-14T11:10:07.734819+0000","last_clean":"2023-07-14T11:10:07.734819+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:10:07.734819+0000","last_undegraded":"2023-07-14T11:10:07.734819+0000","last_fullsized":"2023-07-14T11:10:07.734819+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:26:53.174066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739718+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:10:07.739718+0000","last_peered":"2023-07-14T11:10:07.739718+0000","last_clean":"2023-07-14T11:10:07.739718+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:10:07.739718+0000","last_undegraded":"2023-07-14T11:10:07.739718+0000","last_fullsized":"2023-07-14T11:10:07.739718+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:16:20.330278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'5","reported_seq":74,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:57.443000+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:10:57.443000+0000","last_peered":"2023-07-14T11:10:57.443000+0000","last_clean":"2023-07-14T11:10:57.443000+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:10:57.443000+0000","last_undegraded":"2023-07-14T11:10:57.443000+0000","last_fullsized":"2023-07-14T11:10:57.443000+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:50:46.452985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":43,"num_read_kb":43,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736543+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:10:07.736543+0000","last_peered":"2023-07-14T11:10:07.736543+0000","last_clean":"2023-07-14T11:10:07.736543+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:10:07.736543+0000","last_undegraded":"2023-07-14T11:10:07.736543+0000","last_fullsized":"2023-07-14T11:10:07.736543+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:56:32.791400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738925+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:10:07.738925+0000","last_peered":"2023-07-14T11:10:07.738925+0000","last_clean":"2023-07-14T11:10:07.738925+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:10:07.738925+0000","last_undegraded":"2023-07-14T11:10:07.738925+0000","last_fullsized":"2023-07-14T11:10:07.738925+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:02:28.884240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":35,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:17.382114+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:10:17.382114+0000","last_peered":"2023-07-14T11:10:17.382114+0000","last_clean":"2023-07-14T11:10:17.382114+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:10:17.382114+0000","last_undegraded":"2023-07-14T11:10:17.382114+0000","last_fullsized":"2023-07-14T11:10:17.382114+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:12:29.230412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":354,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736416+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:10:07.736416+0000","last_peered":"2023-07-14T11:10:07.736416+0000","last_clean":"2023-07-14T11:10:07.736416+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:10:07.736416+0000","last_undegraded":"2023-07-14T11:10:07.736416+0000","last_fullsized":"2023-07-14T11:10:07.736416+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:40:53.088449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741053+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:10:07.741053+0000","last_peered":"2023-07-14T11:10:07.741053+0000","last_clean":"2023-07-14T11:10:07.741053+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:10:07.741053+0000","last_undegraded":"2023-07-14T11:10:07.741053+0000","last_fullsized":"2023-07-14T11:10:07.741053+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:46:21.793891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734132+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:10:07.734132+0000","last_peered":"2023-07-14T11:10:07.734132+0000","last_clean":"2023-07-14T11:10:07.734132+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:10:07.734132+0000","last_undegraded":"2023-07-14T11:10:07.734132+0000","last_fullsized":"2023-07-14T11:10:07.734132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:58:44.611438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'4","reported_seq":36,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357605+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:10:14.357605+0000","last_peered":"2023-07-14T11:10:14.357605+0000","last_clean":"2023-07-14T11:10:14.357605+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:10:14.357605+0000","last_undegraded":"2023-07-14T11:10:14.357605+0000","last_fullsized":"2023-07-14T11:10:14.357605+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:06:23.046904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737806+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:10:07.737806+0000","last_peered":"2023-07-14T11:10:07.737806+0000","last_clean":"2023-07-14T11:10:07.737806+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:10:07.737806+0000","last_undegraded":"2023-07-14T11:10:07.737806+0000","last_fullsized":"2023-07-14T11:10:07.737806+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:15.765387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.370106+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:10:15.370106+0000","last_peered":"2023-07-14T11:10:15.370106+0000","last_clean":"2023-07-14T11:10:15.370106+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:10:15.370106+0000","last_undegraded":"2023-07-14T11:10:15.370106+0000","last_fullsized":"2023-07-14T11:10:15.370106+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:27:03.761811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734605+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:10:07.734605+0000","last_peered":"2023-07-14T11:10:07.734605+0000","last_clean":"2023-07-14T11:10:07.734605+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:10:07.734605+0000","last_undegraded":"2023-07-14T11:10:07.734605+0000","last_fullsized":"2023-07-14T11:10:07.734605+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:17:38.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:57.136733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"90'7","reported_seq":43,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369508+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:10:15.369508+0000","last_peered":"2023-07-14T11:10:15.369508+0000","last_clean":"2023-07-14T11:10:15.369508+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:10:15.369508+0000","last_undegraded":"2023-07-14T11:10:15.369508+0000","last_fullsized":"2023-07-14T11:10:15.369508+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:47:01.844602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740024+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:10:07.740024+0000","last_peered":"2023-07-14T11:10:07.740024+0000","last_clean":"2023-07-14T11:10:07.740024+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:10:07.740024+0000","last_undegraded":"2023-07-14T11:10:07.740024+0000","last_fullsized":"2023-07-14T11:10:07.740024+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:34:33.255530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738545+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:10:07.738545+0000","last_peered":"2023-07-14T11:10:07.738545+0000","last_clean":"2023-07-14T11:10:07.738545+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:10:07.738545+0000","last_undegraded":"2023-07-14T11:10:07.738545+0000","last_fullsized":"2023-07-14T11:10:07.738545+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:36.240727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375648+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:10:16.375648+0000","last_peered":"2023-07-14T11:10:16.375648+0000","last_clean":"2023-07-14T11:10:16.375648+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:10:16.375648+0000","last_undegraded":"2023-07-14T11:10:16.375648+0000","last_fullsized":"2023-07-14T11:10:16.375648+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:02:18.139490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141864+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:10:08.141864+0000","last_peered":"2023-07-14T11:10:08.141864+0000","last_clean":"2023-07-14T11:10:08.141864+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:10:08.141864+0000","last_undegraded":"2023-07-14T11:10:08.141864+0000","last_fullsized":"2023-07-14T11:10:08.141864+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:28:33.937520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823662+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:10:07.823662+0000","last_peered":"2023-07-14T11:10:07.823662+0000","last_clean":"2023-07-14T11:10:07.823662+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:10:07.823662+0000","last_undegraded":"2023-07-14T11:10:07.823662+0000","last_fullsized":"2023-07-14T11:10:07.823662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:28:39.306429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.782018+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:10:13.782018+0000","last_peered":"2023-07-14T11:10:13.782018+0000","last_clean":"2023-07-14T11:10:13.782018+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:10:13.782018+0000","last_undegraded":"2023-07-14T11:10:13.782018+0000","last_fullsized":"2023-07-14T11:10:13.782018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:50:14.638370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734307+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:10:07.734307+0000","last_peered":"2023-07-14T11:10:07.734307+0000","last_clean":"2023-07-14T11:10:07.734307+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:10:07.734307+0000","last_undegraded":"2023-07-14T11:10:07.734307+0000","last_fullsized":"2023-07-14T11:10:07.734307+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:27:40.728690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740731+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:10:07.740731+0000","last_peered":"2023-07-14T11:10:07.740731+0000","last_clean":"2023-07-14T11:10:07.740731+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:10:07.740731+0000","last_undegraded":"2023-07-14T11:10:07.740731+0000","last_fullsized":"2023-07-14T11:10:07.740731+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:37.925348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375880+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:10:16.375880+0000","last_peered":"2023-07-14T11:10:16.375880+0000","last_clean":"2023-07-14T11:10:16.375880+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:10:16.375880+0000","last_undegraded":"2023-07-14T11:10:16.375880+0000","last_fullsized":"2023-07-14T11:10:16.375880+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:35:40.777893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737586+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:10:07.737586+0000","last_peered":"2023-07-14T11:10:07.737586+0000","last_clean":"2023-07-14T11:10:07.737586+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:10:07.737586+0000","last_undegraded":"2023-07-14T11:10:07.737586+0000","last_fullsized":"2023-07-14T11:10:07.737586+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:43.286237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823509+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:10:07.823509+0000","last_peered":"2023-07-14T11:10:07.823509+0000","last_clean":"2023-07-14T11:10:07.823509+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:10:07.823509+0000","last_undegraded":"2023-07-14T11:10:07.823509+0000","last_fullsized":"2023-07-14T11:10:07.823509+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:28:35.378022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.843598+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:10:15.843598+0000","last_peered":"2023-07-14T11:10:15.843598+0000","last_clean":"2023-07-14T11:10:15.843598+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:10:15.843598+0000","last_undegraded":"2023-07-14T11:10:15.843598+0000","last_fullsized":"2023-07-14T11:10:15.843598+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:10.923637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739938+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:10:07.739938+0000","last_peered":"2023-07-14T11:10:07.739938+0000","last_clean":"2023-07-14T11:10:07.739938+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:10:07.739938+0000","last_undegraded":"2023-07-14T11:10:07.739938+0000","last_fullsized":"2023-07-14T11:10:07.739938+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:27:36.946955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823764+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:10:07.823764+0000","last_peered":"2023-07-14T11:10:07.823764+0000","last_clean":"2023-07-14T11:10:07.823764+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:10:07.823764+0000","last_undegraded":"2023-07-14T11:10:07.823764+0000","last_fullsized":"2023-07-14T11:10:07.823764+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:46:25.967420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783411+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:10:13.783411+0000","last_peered":"2023-07-14T11:10:13.783411+0000","last_clean":"2023-07-14T11:10:13.783411+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:10:13.783411+0000","last_undegraded":"2023-07-14T11:10:13.783411+0000","last_fullsized":"2023-07-14T11:10:13.783411+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:08:54.674537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736919+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:10:07.736919+0000","last_peered":"2023-07-14T11:10:07.736919+0000","last_clean":"2023-07-14T11:10:07.736919+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:10:07.736919+0000","last_undegraded":"2023-07-14T11:10:07.736919+0000","last_fullsized":"2023-07-14T11:10:07.736919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:34:43.143864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734396+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:10:07.734396+0000","last_peered":"2023-07-14T11:10:07.734396+0000","last_clean":"2023-07-14T11:10:07.734396+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:10:07.734396+0000","last_undegraded":"2023-07-14T11:10:07.734396+0000","last_fullsized":"2023-07-14T11:10:07.734396+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:13:42.740364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"90'5","reported_seq":73,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:55.480151+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:10:55.480151+0000","last_peered":"2023-07-14T11:10:55.480151+0000","last_clean":"2023-07-14T11:10:55.480151+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:10:55.480151+0000","last_undegraded":"2023-07-14T11:10:55.480151+0000","last_fullsized":"2023-07-14T11:10:55.480151+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:17:09.808452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1122,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":42,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737674+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:10:07.737674+0000","last_peered":"2023-07-14T11:10:07.737674+0000","last_clean":"2023-07-14T11:10:07.737674+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:10:07.737674+0000","last_undegraded":"2023-07-14T11:10:07.737674+0000","last_fullsized":"2023-07-14T11:10:07.737674+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:51:08.137181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141293+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:10:08.141293+0000","last_peered":"2023-07-14T11:10:08.141293+0000","last_clean":"2023-07-14T11:10:08.141293+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:10:08.141293+0000","last_undegraded":"2023-07-14T11:10:08.141293+0000","last_fullsized":"2023-07-14T11:10:08.141293+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:33:53.668586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.366487+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:10:15.366487+0000","last_peered":"2023-07-14T11:10:15.366487+0000","last_clean":"2023-07-14T11:10:15.366487+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:10:15.366487+0000","last_undegraded":"2023-07-14T11:10:15.366487+0000","last_fullsized":"2023-07-14T11:10:15.366487+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:41:36.321965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736750+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:10:07.736750+0000","last_peered":"2023-07-14T11:10:07.736750+0000","last_clean":"2023-07-14T11:10:07.736750+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:10:07.736750+0000","last_undegraded":"2023-07-14T11:10:07.736750+0000","last_fullsized":"2023-07-14T11:10:07.736750+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:11:06.323972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739174+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:10:07.739174+0000","last_peered":"2023-07-14T11:10:07.739174+0000","last_clean":"2023-07-14T11:10:07.739174+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:10:07.739174+0000","last_undegraded":"2023-07-14T11:10:07.739174+0000","last_fullsized":"2023-07-14T11:10:07.739174+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:16.916714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"90'5","reported_seq":41,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375147+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:10:16.375147+0000","last_peered":"2023-07-14T11:10:16.375147+0000","last_clean":"2023-07-14T11:10:16.375147+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:10:16.375147+0000","last_undegraded":"2023-07-14T11:10:16.375147+0000","last_fullsized":"2023-07-14T11:10:16.375147+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:33.874908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737919+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:10:07.737919+0000","last_peered":"2023-07-14T11:10:07.737919+0000","last_clean":"2023-07-14T11:10:07.737919+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:10:07.737919+0000","last_undegraded":"2023-07-14T11:10:07.737919+0000","last_fullsized":"2023-07-14T11:10:07.737919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:56.102535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737624+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:10:07.737624+0000","last_peered":"2023-07-14T11:10:07.737624+0000","last_clean":"2023-07-14T11:10:07.737624+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:10:07.737624+0000","last_undegraded":"2023-07-14T11:10:07.737624+0000","last_fullsized":"2023-07-14T11:10:07.737624+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:49.125150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738992+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:10:07.738992+0000","last_peered":"2023-07-14T11:10:07.738992+0000","last_clean":"2023-07-14T11:10:07.738992+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:10:07.738992+0000","last_undegraded":"2023-07-14T11:10:07.738992+0000","last_fullsized":"2023-07-14T11:10:07.738992+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:49:40.242321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738280+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:10:07.738280+0000","last_peered":"2023-07-14T11:10:07.738280+0000","last_clean":"2023-07-14T11:10:07.738280+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:10:07.738280+0000","last_undegraded":"2023-07-14T11:10:07.738280+0000","last_fullsized":"2023-07-14T11:10:07.738280+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:00:40.108545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141672+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:10:08.141672+0000","last_peered":"2023-07-14T11:10:08.141672+0000","last_clean":"2023-07-14T11:10:08.141672+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:10:08.141672+0000","last_undegraded":"2023-07-14T11:10:08.141672+0000","last_fullsized":"2023-07-14T11:10:08.141672+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:35:25.035847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143146+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:10:08.143146+0000","last_peered":"2023-07-14T11:10:08.143146+0000","last_clean":"2023-07-14T11:10:08.143146+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:10:08.143146+0000","last_undegraded":"2023-07-14T11:10:08.143146+0000","last_fullsized":"2023-07-14T11:10:08.143146+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:00:48.749245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744121+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:10:07.744121+0000","last_peered":"2023-07-14T11:10:07.744121+0000","last_clean":"2023-07-14T11:10:07.744121+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:10:07.744121+0000","last_undegraded":"2023-07-14T11:10:07.744121+0000","last_fullsized":"2023-07-14T11:10:07.744121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:56:30.382410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:27.366892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823702+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:10:07.823702+0000","last_peered":"2023-07-14T11:10:07.823702+0000","last_clean":"2023-07-14T11:10:07.823702+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:10:07.823702+0000","last_undegraded":"2023-07-14T11:10:07.823702+0000","last_fullsized":"2023-07-14T11:10:07.823702+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:42.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739599+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:10:07.739599+0000","last_peered":"2023-07-14T11:10:07.739599+0000","last_clean":"2023-07-14T11:10:07.739599+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:10:07.739599+0000","last_undegraded":"2023-07-14T11:10:07.739599+0000","last_fullsized":"2023-07-14T11:10:07.739599+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:09:31.536272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739315+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:10:07.739315+0000","last_peered":"2023-07-14T11:10:07.739315+0000","last_clean":"2023-07-14T11:10:07.739315+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:10:07.739315+0000","last_undegraded":"2023-07-14T11:10:07.739315+0000","last_fullsized":"2023-07-14T11:10:07.739315+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:06:45.531716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734952+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:10:07.734952+0000","last_peered":"2023-07-14T11:10:07.734952+0000","last_clean":"2023-07-14T11:10:07.734952+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:10:07.734952+0000","last_undegraded":"2023-07-14T11:10:07.734952+0000","last_fullsized":"2023-07-14T11:10:07.734952+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:01.416822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734822+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:10:07.734822+0000","last_peered":"2023-07-14T11:10:07.734822+0000","last_clean":"2023-07-14T11:10:07.734822+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:10:07.734822+0000","last_undegraded":"2023-07-14T11:10:07.734822+0000","last_fullsized":"2023-07-14T11:10:07.734822+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:02:57.300787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737927+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:10:07.737927+0000","last_peered":"2023-07-14T11:10:07.737927+0000","last_clean":"2023-07-14T11:10:07.737927+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:10:07.737927+0000","last_undegraded":"2023-07-14T11:10:07.737927+0000","last_fullsized":"2023-07-14T11:10:07.737927+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:53:58.972191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":10024,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":461,"num_read_kb":252,"num_write":209,"num_write_kb":67,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":30072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183,"ondisk_log_size":183,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416412,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.54400000000000004}]},{"osd":1,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.745},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.64900000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.70199999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.67800000000000005}]},{"osd":7,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.75700000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.624},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.73299999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.77900000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.76900000000000002}]}]},{"osd":10,"up_from":72,"seq":309237645343,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28564,"kb_used_data":1612,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716588,"statfs":{"total":95995035648,"available":95965786112,"internally_reserved":0,"allocated":1650688,"data_stored":1084587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":4.0190000000000001,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.90500000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.9419999999999999,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.7210000000000001,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.57299999999999995}]},{"osd":2,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.64500000000000002,"15min":0.64000000000000001},"min":{"1min":0.47499999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.67200000000000004,"15min":0.67000000000000004},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.796,"5min":1.796,"15min":1.796},"last":0.70199999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.876,"15min":0.90400000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9430000000000001,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.67300000000000004,"15min":0.67600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.755}]},{"osd":4,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.56999999999999995,"15min":0.56200000000000006},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.72499999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.014,"5min":1.165,"15min":1.165},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.76900000000000002,"15min":0.77100000000000002},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.9850000000000001,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78100000000000003,"15min":0.78200000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.91800000000000004,"15min":0.94399999999999995},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.5129999999999999,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.96499999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.78100000000000003,"15min":0.78200000000000003},"min":{"1min":0.54800000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.107,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.84199999999999997,"15min":0.85199999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.052,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.94999999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.84999999999999998,"15min":0.84799999999999998},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":2.1970000000000001,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.88300000000000001,"15min":0.89100000000000001},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.6870000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.98799999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.89200000000000002},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":4.8609999999999998,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":1.048}]}]},{"osd":9,"up_from":67,"seq":287762808867,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":1012,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":1036288,"data_stored":489859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.51800000000000002,"15min":0.53100000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.59199999999999997,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.61199999999999999,"15min":0.628},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.94099999999999995,"5min":1.488,"15min":1.488},"last":0.42899999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.76100000000000001,"15min":0.76900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.69799999999999995,"15min":0.70699999999999996},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.89600000000000002,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.74199999999999999,"15min":0.76000000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89900000000000002,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.62,"5min":0.73199999999999998,"15min":0.751},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78900000000000003,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.66600000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.71099999999999997,"15min":0.73099999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.0700000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.69899999999999995,"15min":0.71899999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81499999999999995,"5min":1.292,"15min":1.292},"last":0.625}]},{"osd":4,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.75700000000000001,"15min":0.75700000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.80900000000000005,"15min":0.81699999999999995},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.0529999999999999,"5min":1.165,"15min":1.165},"last":0.64500000000000002}]},{"osd":5,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.84799999999999998,"15min":0.86899999999999999},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.006,"5min":1.242,"15min":1.242},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.81799999999999995,"15min":0.84099999999999997},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.93700000000000006,"5min":1.258,"15min":1.258},"last":0.81799999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.93700000000000006,"15min":0.94899999999999995},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.8069999999999999,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":0.752},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.95699999999999996,"15min":0.97699999999999998},"min":{"1min":0.65100000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7190000000000001,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.72399999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.89200000000000002,"15min":0.91200000000000003},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":0.99199999999999999,"5min":1.331,"15min":1.331},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.97999999999999998,"15min":1.002},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.1240000000000001,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.88400000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.91000000000000003,"15min":0.91600000000000004},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.7310000000000001,"5min":1.7310000000000001,"15min":1.7310000000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.96999999999999997,"15min":0.97599999999999998},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.83999999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.79500000000000004}]}]},{"osd":8,"up_from":59,"seq":253403070504,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":1008,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":1032192,"data_stored":490237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83899999999999997,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59399999999999997,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.84899999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.53000000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.55900000000000005,"15min":0.53600000000000003},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78200000000000003,"5min":0.875,"15min":0.875},"last":0.502},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.61499999999999999,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79800000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.69599999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72299999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.76600000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.40300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93000000000000005,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91800000000000004,"5min":1.079,"15min":1.079},"last":0.63800000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.61899999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.89700000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.64800000000000002,"15min":0.63600000000000001},"min":{"1min":0.56799999999999995,"5min":0.37,"15min":0.37},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.68300000000000005}]},{"osd":5,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.79800000000000004,"5min":1.052,"15min":1.052},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82699999999999996,"5min":1.135,"15min":1.135},"last":0.78400000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.70599999999999996,"15min":0.70799999999999996},"min":{"1min":0.48899999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90100000000000002,"5min":1.008,"15min":1.008},"last":0.745},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.73599999999999999,"15min":0.74199999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.90500000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.71699999999999997},"min":{"1min":0.58199999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":1.028,"15min":1.028},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.73799999999999999,"15min":0.73199999999999998},"min":{"1min":0.54500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.99399999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.83699999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.749,"15min":0.751},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.80500000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.73199999999999998,"15min":0.71999999999999997},"min":{"1min":0.53400000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.873},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75900000000000001,"15min":0.74299999999999999},"min":{"1min":0.65900000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.84999999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.94599999999999995}]}]},{"osd":7,"up_from":51,"seq":219043332141,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28536,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716616,"statfs":{"total":95995035648,"available":95965814784,"internally_reserved":0,"allocated":1622016,"data_stored":1081830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60399999999999998,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.34399999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.91700000000000004,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.625,"15min":0.623},"min":{"1min":0.503,"5min":0.435,"15min":0.435},"max":{"1min":0.85899999999999999,"5min":0.872,"15min":0.872},"last":0.63800000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.61299999999999999,"15min":0.58699999999999997},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.6520000000000001,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.627,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.7690000000000001,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.55900000000000005,"15min":0.54900000000000004},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77600000000000002,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58099999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88700000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.41099999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.71399999999999997,"15min":0.72499999999999998},"min":{"1min":0.48999999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84499999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70899999999999996,"15min":0.71499999999999997},"min":{"1min":0.502,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.75}]},{"osd":6,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.749,"15min":0.73999999999999999},"min":{"1min":0.45900000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.79600000000000004,"15min":0.74399999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":7.3719999999999999,"5min":7.3719999999999999,"15min":7.3719999999999999},"last":0.48899999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.76100000000000001,"15min":0.75900000000000001},"min":{"1min":0.65200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.96599999999999997,"5min":1,"15min":1},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72199999999999998,"15min":0.71999999999999997},"min":{"1min":0.625,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.76900000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80700000000000005,"15min":0.80800000000000005},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0169999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79700000000000004,"15min":0.79300000000000004},"min":{"1min":0.58699999999999997,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.107,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.58999999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.72299999999999998,"15min":0.71799999999999997},"min":{"1min":0.53800000000000003,"5min":0.442,"15min":0.442},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.79200000000000004,"15min":0.79100000000000004},"min":{"1min":0.58499999999999996,"5min":0.37,"15min":0.37},"max":{"1min":1.01,"5min":1.216,"15min":1.216},"last":0.71099999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"min":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.82199999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495666,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":992,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":1015808,"data_stored":489411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48799999999999999,"15min":0.46999999999999997},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.628,"15min":0.60999999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95899999999999996,"5min":1.151,"15min":1.151},"last":0.58699999999999997}]},{"osd":1,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.67900000000000005,"15min":0.67500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90200000000000002,"5min":1.407,"15min":1.407},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.56599999999999995}]},{"osd":2,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74199999999999999,"15min":0.72899999999999998},"min":{"1min":0.60299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.995,"5min":1.087,"15min":1.087},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.70399999999999996,"15min":0.70299999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.91300000000000003}]},{"osd":3,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89000000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.747,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.67600000000000005}]},{"osd":4,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.76000000000000001,"15min":0.745},"min":{"1min":0.505,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.091,"5min":1.173,"15min":1.173},"last":0.438},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0089999999999999,"5min":1.302,"15min":1.302},"last":0.47799999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.72399999999999998,"15min":0.73899999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86599999999999999,"5min":1.004,"15min":1.004},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.82699999999999996,"15min":0.81899999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1890000000000001,"5min":1.264,"15min":1.264},"last":0.65500000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.83299999999999996,"15min":0.80500000000000005},"min":{"1min":0.59599999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2549999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.82899999999999996,"15min":0.81699999999999995},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.157,"5min":1.502,"15min":1.502},"last":0.622}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.64800000000000002,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.93400000000000005,"15min":0.96599999999999997},"min":{"1min":0.62,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2270000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.84999999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.002,"15min":1.0209999999999999},"min":{"1min":0.64400000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2849999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.038},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.0209999999999999,"15min":1.04},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.409,"5min":1.466,"15min":1.466},"last":0.87}]},{"osd":10,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.98199999999999998,"15min":0.98499999999999999},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.304,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.874},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.83099999999999996}]},{"osd":11,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":0.996},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.88400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574922,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27952,"kb_used_data":1000,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717200,"statfs":{"total":95995035648,"available":95966412800,"internally_reserved":0,"allocated":1024000,"data_stored":494235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:22 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.42199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56799999999999995,"15min":0.52000000000000002},"min":{"1min":0.41999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81999999999999995,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.53700000000000003}]},{"osd":2,"last update":"Fri Jul 14 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.625,"15min":0.55700000000000005},"min":{"1min":0.43099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94699999999999995,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.57399999999999995,"15min":0.52200000000000002},"min":{"1min":0.45100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85699999999999998,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.54700000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:09:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.56999999999999995,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77700000000000002,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.64600000000000002,"15min":0.58499999999999996},"min":{"1min":0.57099999999999995,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3819999999999997,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.47999999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.68500000000000005,"15min":0.65000000000000002},"min":{"1min":0.497,"5min":0.38,"15min":0.38},"max":{"1min":0.95499999999999996,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84999999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63300000000000001,"15min":0.60299999999999998},"min":{"1min":0.48999999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.90400000000000003,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.59999999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68000000000000005,"15min":0.66300000000000003},"min":{"1min":0.53600000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86799999999999999,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.72999999999999998},"min":{"1min":0.49399999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1180000000000001,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.63600000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.70899999999999996,"15min":0.67300000000000004},"min":{"1min":0.45000000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.64400000000000002,"15min":0.624},"min":{"1min":0.59699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":0.66900000000000004}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.68999999999999995,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.01,"5min":1.04,"15min":1.04},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.77800000000000002,"15min":0.72999999999999998},"min":{"1min":0.62,"5min":0.441,"15min":0.441},"max":{"1min":1.159,"5min":1.165,"15min":1.165},"last":0.81899999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.754,"15min":0.70399999999999996},"min":{"1min":0.55000000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":5.4260000000000002,"5min":5.4260000000000002,"15min":5.4260000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.752,"15min":0.73399999999999999},"min":{"1min":0.67600000000000005,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.80100000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.85099999999999998,"15min":0.83699999999999997},"min":{"1min":0.57499999999999996,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.875},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.73499999999999999,"15min":0.72099999999999997},"min":{"1min":0.58799999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.70299999999999996}]},{"osd":11,"last update":"Fri Jul 14 11:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"min":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.75}]}]},{"osd":0,"up_from":8,"seq":34359738447,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":1012,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":1036288,"data_stored":490594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.61899999999999999,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.94499999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.629,"15min":0.56699999999999995},"min":{"1min":0.46400000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.86699999999999999,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.48999999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.64100000000000001,"15min":0.57799999999999996},"min":{"1min":0.52400000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1899999999999999,"5min":2.528,"15min":2.528},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.64700000000000002,"15min":0.59899999999999998},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.79500000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":3.2599999999999998,"5min":3.2599999999999998,"15min":3.2599999999999998},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.1500000000000004,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.50900000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.67300000000000004,"15min":0.625},"min":{"1min":0.48399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1120000000000001,"5min":2.48,"15min":2.48},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72899999999999998,"15min":0.68899999999999995},"min":{"1min":0.45200000000000001,"5min":0.443,"15min":0.443},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.72199999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":0.78200000000000003,"15min":0.70299999999999996},"min":{"1min":0.54000000000000004,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.0839999999999996,"5min":6.0839999999999996,"15min":6.0839999999999996},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.71699999999999997,"15min":0.65700000000000003},"min":{"1min":0.50900000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.246,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":1.02}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.70599999999999996,"15min":0.69599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99199999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.71199999999999997,"15min":0.70799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.034,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.42699999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.9239999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.749}]},{"osd":8,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.91100000000000003,"15min":0.88200000000000001},"min":{"1min":0.55700000000000005,"5min":0.496,"15min":0.496},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.88100000000000001,"15min":0.89100000000000001},"min":{"1min":0.68600000000000005,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.91100000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.79700000000000004,"15min":0.79500000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.1000000000000001,"5min":1.232,"15min":1.232},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.89800000000000002,"15min":0.90000000000000002},"min":{"1min":0.49399999999999999,"5min":0.437,"15min":0.437},"max":{"1min":1.2050000000000001,"5min":1.587,"15min":1.587},"last":0.86799999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.879,"15min":0.86799999999999999},"min":{"1min":0.57999999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":2.085,"5min":2.085,"15min":2.085},"last":0.62},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.83899999999999997,"15min":0.82099999999999995},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.97799999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.95499999999999996,"15min":0.90000000000000002},"min":{"1min":0.77200000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":6.0460000000000003,"5min":6.0460000000000003,"15min":6.0460000000000003},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.026,"5min":0.88800000000000001,"15min":0.86499999999999999},"min":{"1min":0.752,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.84899999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411397,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27968,"kb_used_data":1016,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717184,"statfs":{"total":95995035648,"available":95966396416,"internally_reserved":0,"allocated":1040384,"data_stored":494134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54800000000000004,"15min":0.54200000000000004},"min":{"1min":0.41299999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.872},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56100000000000005,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.82999999999999996}]},{"osd":1,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.42299999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.58899999999999997,"15min":0.55600000000000005},"min":{"1min":0.45300000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.68999999999999995}]},{"osd":3,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.60099999999999998,"15min":0.59899999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.83099999999999996,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":1.014},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.63500000000000001,"15min":0.623},"min":{"1min":0.46899999999999997,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.96899999999999997}]},{"osd":4,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.48599999999999999,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73899999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.501,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.84099999999999997,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.63500000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.60899999999999999},"min":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.66200000000000003,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.92300000000000004}]},{"osd":7,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.94799999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.60399999999999998,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.746},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.61499999999999999,"15min":0.58899999999999997},"min":{"1min":0.51800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.84699999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.745,"15min":0.74299999999999999},"min":{"1min":0.63100000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.93300000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.80200000000000005,"15min":0.80000000000000004},"min":{"1min":0.60099999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":1.046}]},{"osd":10,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.78600000000000003,"15min":0.77800000000000002},"min":{"1min":0.52200000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.745,"15min":0.73499999999999999},"min":{"1min":0.55700000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.98499999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":1.002},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":1.032}]}]},{"osd":3,"up_from":25,"seq":107374182463,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27952,"kb_used_data":1000,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717200,"statfs":{"total":95995035648,"available":95966412800,"internally_reserved":0,"allocated":1024000,"data_stored":488894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45200000000000001,"15min":0.40999999999999998},"min":{"1min":0.33800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.66200000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47599999999999998,"15min":0.436},"min":{"1min":0.36099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81299999999999994,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.53500000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.38100000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":1.591},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.65800000000000003,"15min":0.59699999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89700000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.83899999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65800000000000003,"15min":0.63300000000000001},"min":{"1min":0.504,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94099999999999995,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.67400000000000004,"15min":0.64600000000000002},"min":{"1min":0.504,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.046,"5min":1.069,"15min":1.069},"last":0.624}]},{"osd":4,"last update":"Fri Jul 14 11:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.68300000000000005,"15min":0.63800000000000001},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":1.742},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.746,"15min":0.69199999999999995},"min":{"1min":0.48299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.98199999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.75800000000000001,"15min":0.73899999999999999},"min":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.022,"5min":1.262,"15min":1.262},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.75900000000000001,"15min":0.752},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1479999999999999,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.68000000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:10:14 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.752,"15min":0.72999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1739999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.044},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.752,"15min":0.76000000000000001},"min":{"1min":0.501,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.0549999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.1859999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.81799999999999995,"15min":0.82799999999999996},"min":{"1min":0.64600000000000002,"5min":0.496,"15min":0.496},"max":{"1min":1.012,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.2649999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.81699999999999995,"15min":0.81100000000000005},"min":{"1min":0.63500000000000001,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.097,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":1.7050000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.90000000000000002,"15min":0.91100000000000003},"min":{"1min":0.58699999999999997,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.657},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.98599999999999999,"15min":0.996},"min":{"1min":0.64700000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.401,"5min":1.504,"15min":1.504},"last":1.075}]},{"osd":10,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0369999999999999,"15min":1.036},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":4.0300000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":1.23},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.96599999999999997,"15min":0.97399999999999998},"min":{"1min":0.72199999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.1359999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.94699999999999995}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.351,"5min":1.351,"15min":1.351},"last":1.296},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":1.014}]}]},{"osd":4,"up_from":33,"seq":141733920828,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":992,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":1015808,"data_stored":489116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38400000000000001,"15min":0.36599999999999999},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56799999999999995,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40999999999999998,"15min":0.38800000000000001},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61099999999999999,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.29299999999999998}]},{"osd":2,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59999999999999998,"15min":0.57999999999999996},"min":{"1min":0.36799999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.624,"15min":0.58299999999999996},"min":{"1min":0.51200000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.69199999999999995}]},{"osd":3,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.441,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.94999999999999996,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.60199999999999998,"15min":0.58599999999999997},"min":{"1min":0.45100000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.878,"5min":1.054,"15min":1.054},"last":0.60599999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.61399999999999999,"15min":0.58599999999999997},"min":{"1min":0.44800000000000001,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.89400000000000002,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.63600000000000001},"min":{"1min":0.46800000000000003,"5min":0.373,"15min":0.373},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.74299999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.65100000000000002,"15min":0.63500000000000001},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83499999999999996,"5min":1.105,"15min":1.105},"last":0.436},{"interface":"front","average":{"1min":0.753,"5min":0.71299999999999997,"15min":0.68100000000000005},"min":{"1min":0.51000000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.90400000000000003,"5min":1.137,"15min":1.137},"last":0.63}]},{"osd":7,"last update":"Fri Jul 14 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.64900000000000002,"15min":0.61699999999999999},"min":{"1min":0.40400000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.996,"5min":1.087,"15min":1.087},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.623,"15min":0.59599999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.47799999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.73199999999999998,"15min":0.72799999999999998},"min":{"1min":0.54900000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.81000000000000005,"15min":0.80100000000000005},"min":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0329999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.78100000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.83899999999999997,"15min":0.84899999999999998},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.81699999999999995,"15min":0.82499999999999996},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.123,"5min":1.123,"15min":1.123},"last":0.67100000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.1659999999999999,"5min":1.212,"15min":1.212},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.80200000000000005,"15min":0.80400000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.0569999999999999,"5min":1.073,"15min":1.073},"last":0.51100000000000001}]},{"osd":11,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.65200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757303,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28548,"kb_used_data":1596,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716604,"statfs":{"total":95995035648,"available":95965802496,"internally_reserved":0,"allocated":1634304,"data_stored":1081604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.77600000000000002,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.40400000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.78700000000000003,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.42099999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.45800000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73999999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.47899999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82799999999999996,"5min":1.466,"15min":1.466},"last":0.73599999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.437,"15min":0.42799999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.442,"15min":0.435},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.47199999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79100000000000004,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61399999999999999,"15min":0.61499999999999999},"min":{"1min":0.46300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77300000000000002,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.55000000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.70199999999999996,"15min":0.65500000000000003},"min":{"1min":0.51900000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79600000000000004,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.68000000000000005,"15min":0.64000000000000001},"min":{"1min":0.48299999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.71099999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88400000000000001,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68300000000000005,"15min":0.68899999999999995},"min":{"1min":0.51500000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.78000000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70199999999999996,"15min":0.70799999999999996},"min":{"1min":0.55600000000000005,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.86499999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.73599999999999999,"15min":0.73999999999999999},"min":{"1min":0.59499999999999997,"5min":0.496,"15min":0.496},"max":{"1min":1.0489999999999999,"5min":1.528,"15min":1.528},"last":0.57299999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.878,"15min":0.878},"last":0.75700000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.746,"15min":0.74099999999999999},"min":{"1min":0.58799999999999997,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.93000000000000005,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.76100000000000001,"15min":0.75700000000000001},"min":{"1min":0.57499999999999996,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.88,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.72399999999999998}]},{"osd":10,"last update":"Fri Jul 14 11:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.73799999999999999,"15min":0.72999999999999998},"min":{"1min":0.65300000000000002,"5min":0.439,"15min":0.439},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.95099999999999996,"15min":0.97599999999999998},"min":{"1min":0.66200000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.97299999999999998,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.69499999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":2772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":2998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:11:01.272 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph pg dump --format=json 2023-07-14T11:11:01.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:01 smithi130 ceph-mon[129405]: pgmap v317: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:01.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:01 smithi130 ceph-mon[129405]: from='client.25006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:01.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:01 smithi040 ceph-mon[97954]: pgmap v317: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:01.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:01 smithi040 ceph-mon[97954]: from='client.25006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:01.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:01 smithi045 ceph-mon[128791]: pgmap v317: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:01 smithi045 ceph-mon[128791]: from='client.25006 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:02.033 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:03.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 -- 172.21.15.40:0/2036658541 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac10c3e0 msgr2=0x7fdfac10e7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:03.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/2036658541 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac10c3e0 0x7fdfac10e7c0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7fdfa400b0a0 tx=0x7fdfa400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 -- 172.21.15.40:0/2036658541 shutdown_connections 2023-07-14T11:11:03.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/2036658541 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac10c3e0 0x7fdfac10e7c0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/2036658541 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/2036658541 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac0731c0 0x7fdfac10bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 -- 172.21.15.40:0/2036658541 >> 172.21.15.40:0/2036658541 conn(0x7fdfac0fb660 msgr2=0x7fdfac0fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:03.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 -- 172.21.15.40:0/2036658541 shutdown_connections 2023-07-14T11:11:03.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.535+0000 7fdfb11e6700 1 -- 172.21.15.40:0/2036658541 wait complete. 2023-07-14T11:11:03.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 Processor -- start 2023-07-14T11:11:03.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 -- start start 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac197ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 0x7fdfac19c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdfac19cf30 con 0x7fdfac0731c0 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdfac19d0a0 con 0x7fdfac0728b0 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.536+0000 7fdfb11e6700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdfac19d260 con 0x7fdfac10c3e0 2023-07-14T11:11:03.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42710/0 (socket says 172.21.15.40:42710) 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 -- 172.21.15.40:0/3355143948 learned_addr learned my addr 172.21.15.40:0/3355143948 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaad9d700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac197ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfab59e700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 0x7fdfac19c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 -- 172.21.15.40:0/3355143948 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 msgr2=0x7fdfac19c890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 0x7fdfac19c890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 -- 172.21.15.40:0/3355143948 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 msgr2=0x7fdfac197ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac197ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 -- 172.21.15.40:0/3355143948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdfa4009d00 con 0x7fdfac0731c0 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfab59e700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 0x7fdfac19c890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaad9d700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.537+0000 7fdfaa59c700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fdfa000e9a0 tx=0x7fdfa000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.538+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfa0009790 con 0x7fdfac0731c0 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.538+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdfa0010430 con 0x7fdfac0731c0 2023-07-14T11:11:03.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.538+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdfac19d540 con 0x7fdfac0731c0 2023-07-14T11:11:03.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.538+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdfa0018680 con 0x7fdfac0731c0 2023-07-14T11:11:03.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.538+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdfac1a4d10 con 0x7fdfac0731c0 2023-07-14T11:11:03.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.539+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdfac106d00 con 0x7fdfac0731c0 2023-07-14T11:11:03.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.541+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fdfa00105a0 con 0x7fdfac0731c0 2023-07-14T11:11:03.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.543+0000 7fdf93fff700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 0x7fdf940a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:03.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.543+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fdfa0014070 con 0x7fdfac0731c0 2023-07-14T11:11:03.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.543+0000 7fdfaad9d700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 0x7fdf940a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:03.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.544+0000 7fdfaad9d700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 0x7fdf940a3600 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fdf9c011490 tx=0x7fdf9c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:03.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.545+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdfa0096360 con 0x7fdfac0731c0 2023-07-14T11:11:03.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.751+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fdfac02c8c0 con 0x7fdf940a1150 2023-07-14T11:11:03.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.764+0000 7fdf93fff700 1 -- 172.21.15.40:0/3355143948 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+426773 (secure 0 0 0) 0x7fdfac02c8c0 con 0x7fdf940a1150 2023-07-14T11:11:03.773 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:03.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 msgr2=0x7fdf940a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:03.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 0x7fdf940a3600 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fdf9c011490 tx=0x7fdf9c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 msgr2=0x7fdfac198420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7fdfa000e9a0 tx=0x7fdfa000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 shutdown_connections 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf940a1150 0x7fdf940a3600 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdfac10c3e0 0x7fdfac19c890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdfac0728b0 0x7fdfac197ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 --2- 172.21.15.40:0/3355143948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdfac0731c0 0x7fdfac198420 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:03.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.770+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 >> 172.21.15.40:0/3355143948 conn(0x7fdfac0fb660 msgr2=0x7fdfac103e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:03.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.771+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 shutdown_connections 2023-07-14T11:11:03.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:03.771+0000 7fdfb11e6700 1 -- 172.21.15.40:0/3355143948 wait complete. 2023-07-14T11:11:03.777 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:11:03.962 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:03 smithi040 ceph-mon[97954]: pgmap v318: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:03 smithi130 ceph-mon[129405]: pgmap v318: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:03 smithi045 ceph-mon[128791]: pgmap v318: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:04.330 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":318,"stamp":"2023-07-14T11:11:02.645458+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":600392,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":617,"num_read_kb":376,"num_write":390,"num_write_kb":3005,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":320,"ondisk_log_size":320,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337236,"kb_used_data":13812,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604588,"statfs":{"total":1151940427776,"available":1151595098112,"internally_reserved":0,"allocated":14143488,"data_stored":7664382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":120,"num_read_kb":55,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003491"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738359+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:10:07.738359+0000","last_peered":"2023-07-14T11:10:07.738359+0000","last_clean":"2023-07-14T11:10:07.738359+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:10:07.738359+0000","last_undegraded":"2023-07-14T11:10:07.738359+0000","last_fullsized":"2023-07-14T11:10:07.738359+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:21:10.616471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141620+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:10:08.141620+0000","last_peered":"2023-07-14T11:10:08.141620+0000","last_clean":"2023-07-14T11:10:08.141620+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:10:08.141620+0000","last_undegraded":"2023-07-14T11:10:08.141620+0000","last_fullsized":"2023-07-14T11:10:08.141620+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:43:13.354815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142202+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:10:08.142202+0000","last_peered":"2023-07-14T11:10:08.142202+0000","last_clean":"2023-07-14T11:10:08.142202+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:10:08.142202+0000","last_undegraded":"2023-07-14T11:10:08.142202+0000","last_fullsized":"2023-07-14T11:10:08.142202+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:25:21.721235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823979+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:10:07.823979+0000","last_peered":"2023-07-14T11:10:07.823979+0000","last_clean":"2023-07-14T11:10:07.823979+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:10:07.823979+0000","last_undegraded":"2023-07-14T11:10:07.823979+0000","last_fullsized":"2023-07-14T11:10:07.823979+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:50:39.156238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734553+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:10:07.734553+0000","last_peered":"2023-07-14T11:10:07.734553+0000","last_clean":"2023-07-14T11:10:07.734553+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:10:07.734553+0000","last_undegraded":"2023-07-14T11:10:07.734553+0000","last_fullsized":"2023-07-14T11:10:07.734553+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:08:20.010349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142872+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:10:08.142872+0000","last_peered":"2023-07-14T11:10:08.142872+0000","last_clean":"2023-07-14T11:10:08.142872+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:10:08.142872+0000","last_undegraded":"2023-07-14T11:10:08.142872+0000","last_fullsized":"2023-07-14T11:10:08.142872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:38:58.846023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736546+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:10:07.736546+0000","last_peered":"2023-07-14T11:10:07.736546+0000","last_clean":"2023-07-14T11:10:07.736546+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:10:07.736546+0000","last_undegraded":"2023-07-14T11:10:07.736546+0000","last_fullsized":"2023-07-14T11:10:07.736546+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:29:36.124497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739054+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:10:07.739054+0000","last_peered":"2023-07-14T11:10:07.739054+0000","last_clean":"2023-07-14T11:10:07.739054+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:10:07.739054+0000","last_undegraded":"2023-07-14T11:10:07.739054+0000","last_fullsized":"2023-07-14T11:10:07.739054+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:48:52.238878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735166+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:10:07.735166+0000","last_peered":"2023-07-14T11:10:07.735166+0000","last_clean":"2023-07-14T11:10:07.735166+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:10:07.735166+0000","last_undegraded":"2023-07-14T11:10:07.735166+0000","last_fullsized":"2023-07-14T11:10:07.735166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:45.770708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143166+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:10:08.143166+0000","last_peered":"2023-07-14T11:10:08.143166+0000","last_clean":"2023-07-14T11:10:08.143166+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:10:08.143166+0000","last_undegraded":"2023-07-14T11:10:08.143166+0000","last_fullsized":"2023-07-14T11:10:08.143166+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:11.980357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737468+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:10:07.737468+0000","last_peered":"2023-07-14T11:10:07.737468+0000","last_clean":"2023-07-14T11:10:07.737468+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:10:07.737468+0000","last_undegraded":"2023-07-14T11:10:07.737468+0000","last_fullsized":"2023-07-14T11:10:07.737468+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:31:39.003266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823717+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:10:07.823717+0000","last_peered":"2023-07-14T11:10:07.823717+0000","last_clean":"2023-07-14T11:10:07.823717+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:10:07.823717+0000","last_undegraded":"2023-07-14T11:10:07.823717+0000","last_fullsized":"2023-07-14T11:10:07.823717+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:03:29.844122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734676+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:10:07.734676+0000","last_peered":"2023-07-14T11:10:07.734676+0000","last_clean":"2023-07-14T11:10:07.734676+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:10:07.734676+0000","last_undegraded":"2023-07-14T11:10:07.734676+0000","last_fullsized":"2023-07-14T11:10:07.734676+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:10:43.308729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141270+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:10:08.141270+0000","last_peered":"2023-07-14T11:10:08.141270+0000","last_clean":"2023-07-14T11:10:08.141270+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:10:08.141270+0000","last_undegraded":"2023-07-14T11:10:08.141270+0000","last_fullsized":"2023-07-14T11:10:08.141270+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:20:00.054130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141301+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:10:08.141301+0000","last_peered":"2023-07-14T11:10:08.141301+0000","last_clean":"2023-07-14T11:10:08.141301+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:10:08.141301+0000","last_undegraded":"2023-07-14T11:10:08.141301+0000","last_fullsized":"2023-07-14T11:10:08.141301+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:38:22.587911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734282+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:10:07.734282+0000","last_peered":"2023-07-14T11:10:07.734282+0000","last_clean":"2023-07-14T11:10:07.734282+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:10:07.734282+0000","last_undegraded":"2023-07-14T11:10:07.734282+0000","last_fullsized":"2023-07-14T11:10:07.734282+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:53:57.650717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738135+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:10:07.738135+0000","last_peered":"2023-07-14T11:10:07.738135+0000","last_clean":"2023-07-14T11:10:07.738135+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:10:07.738135+0000","last_undegraded":"2023-07-14T11:10:07.738135+0000","last_fullsized":"2023-07-14T11:10:07.738135+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:48:17.654653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740775+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:10:07.740775+0000","last_peered":"2023-07-14T11:10:07.740775+0000","last_clean":"2023-07-14T11:10:07.740775+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:10:07.740775+0000","last_undegraded":"2023-07-14T11:10:07.740775+0000","last_fullsized":"2023-07-14T11:10:07.740775+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:36:51.637622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737923+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:10:07.737923+0000","last_peered":"2023-07-14T11:10:07.737923+0000","last_clean":"2023-07-14T11:10:07.737923+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:07.737923+0000","last_undegraded":"2023-07-14T11:10:07.737923+0000","last_fullsized":"2023-07-14T11:10:07.737923+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:45.243063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738752+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:10:07.738752+0000","last_peered":"2023-07-14T11:10:07.738752+0000","last_clean":"2023-07-14T11:10:07.738752+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:10:07.738752+0000","last_undegraded":"2023-07-14T11:10:07.738752+0000","last_fullsized":"2023-07-14T11:10:07.738752+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:12:05.905433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"90'13","reported_seq":85,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:11:00.466999+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:11:00.466999+0000","last_peered":"2023-07-14T11:11:00.466999+0000","last_clean":"2023-07-14T11:11:00.466999+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:11:00.466999+0000","last_undegraded":"2023-07-14T11:11:00.466999+0000","last_fullsized":"2023-07-14T11:11:00.466999+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:21:25.983167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":46,"num_read_kb":46,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:53.251986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822942+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:10:07.822942+0000","last_peered":"2023-07-14T11:10:07.822942+0000","last_clean":"2023-07-14T11:10:07.822942+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:10:07.822942+0000","last_undegraded":"2023-07-14T11:10:07.822942+0000","last_fullsized":"2023-07-14T11:10:07.822942+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:32:00.530574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"90'2","reported_seq":28,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:56.375572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142002+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:10:08.142002+0000","last_peered":"2023-07-14T11:10:08.142002+0000","last_clean":"2023-07-14T11:10:08.142002+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:10:08.142002+0000","last_undegraded":"2023-07-14T11:10:08.142002+0000","last_fullsized":"2023-07-14T11:10:08.142002+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:26.963699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823419+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:10:07.823419+0000","last_peered":"2023-07-14T11:10:07.823419+0000","last_clean":"2023-07-14T11:10:07.823419+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:10:07.823419+0000","last_undegraded":"2023-07-14T11:10:07.823419+0000","last_fullsized":"2023-07-14T11:10:07.823419+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:43.101537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'12","reported_seq":39,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:17.382526+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:10:17.382526+0000","last_peered":"2023-07-14T11:10:17.382526+0000","last_clean":"2023-07-14T11:10:17.382526+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:10:17.382526+0000","last_undegraded":"2023-07-14T11:10:17.382526+0000","last_fullsized":"2023-07-14T11:10:17.382526+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:26:07.506765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734736+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:10:07.734736+0000","last_peered":"2023-07-14T11:10:07.734736+0000","last_clean":"2023-07-14T11:10:07.734736+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:10:07.734736+0000","last_undegraded":"2023-07-14T11:10:07.734736+0000","last_fullsized":"2023-07-14T11:10:07.734736+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:10:30.512188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.822955+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:10:07.822955+0000","last_peered":"2023-07-14T11:10:07.822955+0000","last_clean":"2023-07-14T11:10:07.822955+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:10:07.822955+0000","last_undegraded":"2023-07-14T11:10:07.822955+0000","last_fullsized":"2023-07-14T11:10:07.822955+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:44.974006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376665+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:10:16.376665+0000","last_peered":"2023-07-14T11:10:16.376665+0000","last_clean":"2023-07-14T11:10:16.376665+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:10:16.376665+0000","last_undegraded":"2023-07-14T11:10:16.376665+0000","last_fullsized":"2023-07-14T11:10:16.376665+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:41:26.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740246+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:10:07.740246+0000","last_peered":"2023-07-14T11:10:07.740246+0000","last_clean":"2023-07-14T11:10:07.740246+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:10:07.740246+0000","last_undegraded":"2023-07-14T11:10:07.740246+0000","last_fullsized":"2023-07-14T11:10:07.740246+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:32.963491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738142+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:10:07.738142+0000","last_peered":"2023-07-14T11:10:07.738142+0000","last_clean":"2023-07-14T11:10:07.738142+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:10:07.738142+0000","last_undegraded":"2023-07-14T11:10:07.738142+0000","last_fullsized":"2023-07-14T11:10:07.738142+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:51:58.731259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"90'5","reported_seq":81,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:11:00.426236+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:11:00.426236+0000","last_peered":"2023-07-14T11:11:00.426236+0000","last_clean":"2023-07-14T11:11:00.426236+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:11:00.426236+0000","last_undegraded":"2023-07-14T11:11:00.426236+0000","last_fullsized":"2023-07-14T11:11:00.426236+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:06:48.309845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":50,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734873+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:10:07.734873+0000","last_peered":"2023-07-14T11:10:07.734873+0000","last_clean":"2023-07-14T11:10:07.734873+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:10:07.734873+0000","last_undegraded":"2023-07-14T11:10:07.734873+0000","last_fullsized":"2023-07-14T11:10:07.734873+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:48.090320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738348+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:10:07.738348+0000","last_peered":"2023-07-14T11:10:07.738348+0000","last_clean":"2023-07-14T11:10:07.738348+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:10:07.738348+0000","last_undegraded":"2023-07-14T11:10:07.738348+0000","last_fullsized":"2023-07-14T11:10:07.738348+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:36:58.104207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369266+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:10:15.369266+0000","last_peered":"2023-07-14T11:10:15.369266+0000","last_clean":"2023-07-14T11:10:15.369266+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:10:15.369266+0000","last_undegraded":"2023-07-14T11:10:15.369266+0000","last_fullsized":"2023-07-14T11:10:15.369266+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:38:46.335485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.745285+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:10:07.745285+0000","last_peered":"2023-07-14T11:10:07.745285+0000","last_clean":"2023-07-14T11:10:07.745285+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:10:07.745285+0000","last_undegraded":"2023-07-14T11:10:07.745285+0000","last_fullsized":"2023-07-14T11:10:07.745285+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:17.289951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737973+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:10:07.737973+0000","last_peered":"2023-07-14T11:10:07.737973+0000","last_clean":"2023-07-14T11:10:07.737973+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:10:07.737973+0000","last_undegraded":"2023-07-14T11:10:07.737973+0000","last_fullsized":"2023-07-14T11:10:07.737973+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:07:16.888150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.360843+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:10:14.360843+0000","last_peered":"2023-07-14T11:10:14.360843+0000","last_clean":"2023-07-14T11:10:14.360843+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:10:14.360843+0000","last_undegraded":"2023-07-14T11:10:14.360843+0000","last_fullsized":"2023-07-14T11:10:14.360843+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:53:35.630621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738872+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:10:07.738872+0000","last_peered":"2023-07-14T11:10:07.738872+0000","last_clean":"2023-07-14T11:10:07.738872+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:10:07.738872+0000","last_undegraded":"2023-07-14T11:10:07.738872+0000","last_fullsized":"2023-07-14T11:10:07.738872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:01:08.293055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141456+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:10:08.141456+0000","last_peered":"2023-07-14T11:10:08.141456+0000","last_clean":"2023-07-14T11:10:08.141456+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:10:08.141456+0000","last_undegraded":"2023-07-14T11:10:08.141456+0000","last_fullsized":"2023-07-14T11:10:08.141456+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:14:59.191235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375874+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:10:16.375874+0000","last_peered":"2023-07-14T11:10:16.375874+0000","last_clean":"2023-07-14T11:10:16.375874+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:10:16.375874+0000","last_undegraded":"2023-07-14T11:10:16.375874+0000","last_fullsized":"2023-07-14T11:10:16.375874+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:39:15.619767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738091+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:10:07.738091+0000","last_peered":"2023-07-14T11:10:07.738091+0000","last_clean":"2023-07-14T11:10:07.738091+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:10:07.738091+0000","last_undegraded":"2023-07-14T11:10:07.738091+0000","last_fullsized":"2023-07-14T11:10:07.738091+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:48:41.840080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739132+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:10:07.739132+0000","last_peered":"2023-07-14T11:10:07.739132+0000","last_clean":"2023-07-14T11:10:07.739132+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:10:07.739132+0000","last_undegraded":"2023-07-14T11:10:07.739132+0000","last_fullsized":"2023-07-14T11:10:07.739132+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:39:26.932728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'4","reported_seq":119,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:59.424262+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:10:59.424262+0000","last_peered":"2023-07-14T11:10:59.424262+0000","last_clean":"2023-07-14T11:10:59.424262+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:10:59.424262+0000","last_undegraded":"2023-07-14T11:10:59.424262+0000","last_fullsized":"2023-07-14T11:10:59.424262+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:10:05.049869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":89,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735216+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:10:07.735216+0000","last_peered":"2023-07-14T11:10:07.735216+0000","last_clean":"2023-07-14T11:10:07.735216+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:10:07.735216+0000","last_undegraded":"2023-07-14T11:10:07.735216+0000","last_fullsized":"2023-07-14T11:10:07.735216+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:50.720818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737634+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:10:07.737634+0000","last_peered":"2023-07-14T11:10:07.737634+0000","last_clean":"2023-07-14T11:10:07.737634+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:10:07.737634+0000","last_undegraded":"2023-07-14T11:10:07.737634+0000","last_fullsized":"2023-07-14T11:10:07.737634+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:25:31.401998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'4","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375310+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:10:16.375310+0000","last_peered":"2023-07-14T11:10:16.375310+0000","last_clean":"2023-07-14T11:10:16.375310+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:10:16.375310+0000","last_undegraded":"2023-07-14T11:10:16.375310+0000","last_fullsized":"2023-07-14T11:10:16.375310+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:10:40.132095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142089+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:10:08.142089+0000","last_peered":"2023-07-14T11:10:08.142089+0000","last_clean":"2023-07-14T11:10:08.142089+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:10:08.142089+0000","last_undegraded":"2023-07-14T11:10:08.142089+0000","last_fullsized":"2023-07-14T11:10:08.142089+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:56:07.808822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823945+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:10:07.823945+0000","last_peered":"2023-07-14T11:10:07.823945+0000","last_clean":"2023-07-14T11:10:07.823945+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:10:07.823945+0000","last_undegraded":"2023-07-14T11:10:07.823945+0000","last_fullsized":"2023-07-14T11:10:07.823945+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:16:13.828703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.351487+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:10:14.351487+0000","last_peered":"2023-07-14T11:10:14.351487+0000","last_clean":"2023-07-14T11:10:14.351487+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:10:14.351487+0000","last_undegraded":"2023-07-14T11:10:14.351487+0000","last_fullsized":"2023-07-14T11:10:14.351487+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:30:17.168629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738962+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:10:07.738962+0000","last_peered":"2023-07-14T11:10:07.738962+0000","last_clean":"2023-07-14T11:10:07.738962+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:10:07.738962+0000","last_undegraded":"2023-07-14T11:10:07.738962+0000","last_fullsized":"2023-07-14T11:10:07.738962+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:25:22.187958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737948+0000","last_change":"2023-07-14T11:09:47.624060+0000","last_active":"2023-07-14T11:10:07.737948+0000","last_peered":"2023-07-14T11:10:07.737948+0000","last_clean":"2023-07-14T11:10:07.737948+0000","last_became_active":"2023-07-14T11:09:47.623605+0000","last_became_peered":"2023-07-14T11:09:47.623605+0000","last_unstale":"2023-07-14T11:10:07.737948+0000","last_undegraded":"2023-07-14T11:10:07.737948+0000","last_fullsized":"2023-07-14T11:10:07.737948+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:17:15.881693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,8,11,5],"acting":[9,8,11,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.14","version":"90'14","reported_seq":134,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:59.460658+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:10:59.460658+0000","last_peered":"2023-07-14T11:10:59.460658+0000","last_clean":"2023-07-14T11:10:59.460658+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:10:59.460658+0000","last_undegraded":"2023-07-14T11:10:59.460658+0000","last_fullsized":"2023-07-14T11:10:59.460658+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:01:52.717646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3794,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":94,"num_read_kb":51,"num_write":16,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734684+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:10:07.734684+0000","last_peered":"2023-07-14T11:10:07.734684+0000","last_clean":"2023-07-14T11:10:07.734684+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:10:07.734684+0000","last_undegraded":"2023-07-14T11:10:07.734684+0000","last_fullsized":"2023-07-14T11:10:07.734684+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:55.540298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740741+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:10:07.740741+0000","last_peered":"2023-07-14T11:10:07.740741+0000","last_clean":"2023-07-14T11:10:07.740741+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:10:07.740741+0000","last_undegraded":"2023-07-14T11:10:07.740741+0000","last_fullsized":"2023-07-14T11:10:07.740741+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:37.113063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"90'9","reported_seq":37,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.376065+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:10:16.376065+0000","last_peered":"2023-07-14T11:10:16.376065+0000","last_clean":"2023-07-14T11:10:16.376065+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:10:16.376065+0000","last_undegraded":"2023-07-14T11:10:16.376065+0000","last_fullsized":"2023-07-14T11:10:16.376065+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:39:38.915458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734456+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:10:07.734456+0000","last_peered":"2023-07-14T11:10:07.734456+0000","last_clean":"2023-07-14T11:10:07.734456+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:10:07.734456+0000","last_undegraded":"2023-07-14T11:10:07.734456+0000","last_fullsized":"2023-07-14T11:10:07.734456+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:25:56.604869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744522+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:10:07.744522+0000","last_peered":"2023-07-14T11:10:07.744522+0000","last_clean":"2023-07-14T11:10:07.744522+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:10:07.744522+0000","last_undegraded":"2023-07-14T11:10:07.744522+0000","last_fullsized":"2023-07-14T11:10:07.744522+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:08.719446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"90'5","reported_seq":32,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375209+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:10:16.375209+0000","last_peered":"2023-07-14T11:10:16.375209+0000","last_clean":"2023-07-14T11:10:16.375209+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:10:16.375209+0000","last_undegraded":"2023-07-14T11:10:16.375209+0000","last_fullsized":"2023-07-14T11:10:16.375209+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:12:45.811503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736758+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:10:07.736758+0000","last_peered":"2023-07-14T11:10:07.736758+0000","last_clean":"2023-07-14T11:10:07.736758+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:10:07.736758+0000","last_undegraded":"2023-07-14T11:10:07.736758+0000","last_fullsized":"2023-07-14T11:10:07.736758+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:28.882720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740995+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:10:07.740995+0000","last_peered":"2023-07-14T11:10:07.740995+0000","last_clean":"2023-07-14T11:10:07.740995+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:10:07.740995+0000","last_undegraded":"2023-07-14T11:10:07.740995+0000","last_fullsized":"2023-07-14T11:10:07.740995+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:35.770995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":78,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:11:00.448243+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:11:00.448243+0000","last_peered":"2023-07-14T11:11:00.448243+0000","last_clean":"2023-07-14T11:11:00.448243+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:11:00.448243+0000","last_undegraded":"2023-07-14T11:11:00.448243+0000","last_fullsized":"2023-07-14T11:11:00.448243+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:57:32.688631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":48,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737387+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:10:07.737387+0000","last_peered":"2023-07-14T11:10:07.737387+0000","last_clean":"2023-07-14T11:10:07.737387+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:10:07.737387+0000","last_undegraded":"2023-07-14T11:10:07.737387+0000","last_fullsized":"2023-07-14T11:10:07.737387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:18:06.626041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734847+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:10:07.734847+0000","last_peered":"2023-07-14T11:10:07.734847+0000","last_clean":"2023-07-14T11:10:07.734847+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:10:07.734847+0000","last_undegraded":"2023-07-14T11:10:07.734847+0000","last_fullsized":"2023-07-14T11:10:07.734847+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:14:05.500061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":44,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375086+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:10:16.375086+0000","last_peered":"2023-07-14T11:10:16.375086+0000","last_clean":"2023-07-14T11:10:16.375086+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:10:16.375086+0000","last_undegraded":"2023-07-14T11:10:16.375086+0000","last_fullsized":"2023-07-14T11:10:16.375086+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:26:26.993318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741882+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:10:07.741882+0000","last_peered":"2023-07-14T11:10:07.741882+0000","last_clean":"2023-07-14T11:10:07.741882+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:10:07.741882+0000","last_undegraded":"2023-07-14T11:10:07.741882+0000","last_fullsized":"2023-07-14T11:10:07.741882+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:38:12.515688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741815+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:10:07.741815+0000","last_peered":"2023-07-14T11:10:07.741815+0000","last_clean":"2023-07-14T11:10:07.741815+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:10:07.741815+0000","last_undegraded":"2023-07-14T11:10:07.741815+0000","last_fullsized":"2023-07-14T11:10:07.741815+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:39.325247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"90'2","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.349371+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:10:14.349371+0000","last_peered":"2023-07-14T11:10:14.349371+0000","last_clean":"2023-07-14T11:10:14.349371+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:10:14.349371+0000","last_undegraded":"2023-07-14T11:10:14.349371+0000","last_fullsized":"2023-07-14T11:10:14.349371+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:16.215502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738122+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:10:07.738122+0000","last_peered":"2023-07-14T11:10:07.738122+0000","last_clean":"2023-07-14T11:10:07.738122+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:10:07.738122+0000","last_undegraded":"2023-07-14T11:10:07.738122+0000","last_fullsized":"2023-07-14T11:10:07.738122+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:28.226381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734949+0000","last_change":"2023-07-14T11:09:47.736950+0000","last_active":"2023-07-14T11:10:07.734949+0000","last_peered":"2023-07-14T11:10:07.734949+0000","last_clean":"2023-07-14T11:10:07.734949+0000","last_became_active":"2023-07-14T11:09:47.736506+0000","last_became_peered":"2023-07-14T11:09:47.736506+0000","last_unstale":"2023-07-14T11:10:07.734949+0000","last_undegraded":"2023-07-14T11:10:07.734949+0000","last_fullsized":"2023-07-14T11:10:07.734949+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:24.567987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,9],"acting":[8,10,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"90'1","reported_seq":27,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:11:19.303557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.142811+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:10:08.142811+0000","last_peered":"2023-07-14T11:10:08.142811+0000","last_clean":"2023-07-14T11:10:08.142811+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:10:08.142811+0000","last_undegraded":"2023-07-14T11:10:08.142811+0000","last_fullsized":"2023-07-14T11:10:08.142811+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:51:22.856363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.735220+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:10:07.735220+0000","last_peered":"2023-07-14T11:10:07.735220+0000","last_clean":"2023-07-14T11:10:07.735220+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:10:07.735220+0000","last_undegraded":"2023-07-14T11:10:07.735220+0000","last_fullsized":"2023-07-14T11:10:07.735220+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:52:35.457076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'5","reported_seq":75,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:58.464547+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:10:58.464547+0000","last_peered":"2023-07-14T11:10:58.464547+0000","last_clean":"2023-07-14T11:10:58.464547+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:10:58.464547+0000","last_undegraded":"2023-07-14T11:10:58.464547+0000","last_fullsized":"2023-07-14T11:10:58.464547+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:49:44.032115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143050+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:10:08.143050+0000","last_peered":"2023-07-14T11:10:08.143050+0000","last_clean":"2023-07-14T11:10:08.143050+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:10:08.143050+0000","last_undegraded":"2023-07-14T11:10:08.143050+0000","last_fullsized":"2023-07-14T11:10:08.143050+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:04:11.876243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734819+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:10:07.734819+0000","last_peered":"2023-07-14T11:10:07.734819+0000","last_clean":"2023-07-14T11:10:07.734819+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:10:07.734819+0000","last_undegraded":"2023-07-14T11:10:07.734819+0000","last_fullsized":"2023-07-14T11:10:07.734819+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:26:53.174066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739718+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:10:07.739718+0000","last_peered":"2023-07-14T11:10:07.739718+0000","last_clean":"2023-07-14T11:10:07.739718+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:10:07.739718+0000","last_undegraded":"2023-07-14T11:10:07.739718+0000","last_fullsized":"2023-07-14T11:10:07.739718+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:16:20.330278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'5","reported_seq":74,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:57.443000+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:10:57.443000+0000","last_peered":"2023-07-14T11:10:57.443000+0000","last_clean":"2023-07-14T11:10:57.443000+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:10:57.443000+0000","last_undegraded":"2023-07-14T11:10:57.443000+0000","last_fullsized":"2023-07-14T11:10:57.443000+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:50:46.452985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":43,"num_read_kb":43,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736543+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:10:07.736543+0000","last_peered":"2023-07-14T11:10:07.736543+0000","last_clean":"2023-07-14T11:10:07.736543+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:10:07.736543+0000","last_undegraded":"2023-07-14T11:10:07.736543+0000","last_fullsized":"2023-07-14T11:10:07.736543+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:56:32.791400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738925+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:10:07.738925+0000","last_peered":"2023-07-14T11:10:07.738925+0000","last_clean":"2023-07-14T11:10:07.738925+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:10:07.738925+0000","last_undegraded":"2023-07-14T11:10:07.738925+0000","last_fullsized":"2023-07-14T11:10:07.738925+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:02:28.884240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":35,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:17.382114+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:10:17.382114+0000","last_peered":"2023-07-14T11:10:17.382114+0000","last_clean":"2023-07-14T11:10:17.382114+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:10:17.382114+0000","last_undegraded":"2023-07-14T11:10:17.382114+0000","last_fullsized":"2023-07-14T11:10:17.382114+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:12:29.230412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":354,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736416+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:10:07.736416+0000","last_peered":"2023-07-14T11:10:07.736416+0000","last_clean":"2023-07-14T11:10:07.736416+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:10:07.736416+0000","last_undegraded":"2023-07-14T11:10:07.736416+0000","last_fullsized":"2023-07-14T11:10:07.736416+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:40:53.088449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.741053+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:10:07.741053+0000","last_peered":"2023-07-14T11:10:07.741053+0000","last_clean":"2023-07-14T11:10:07.741053+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:10:07.741053+0000","last_undegraded":"2023-07-14T11:10:07.741053+0000","last_fullsized":"2023-07-14T11:10:07.741053+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:46:21.793891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734132+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:10:07.734132+0000","last_peered":"2023-07-14T11:10:07.734132+0000","last_clean":"2023-07-14T11:10:07.734132+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:10:07.734132+0000","last_undegraded":"2023-07-14T11:10:07.734132+0000","last_fullsized":"2023-07-14T11:10:07.734132+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:58:44.611438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'4","reported_seq":36,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:14.357605+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:10:14.357605+0000","last_peered":"2023-07-14T11:10:14.357605+0000","last_clean":"2023-07-14T11:10:14.357605+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:10:14.357605+0000","last_undegraded":"2023-07-14T11:10:14.357605+0000","last_fullsized":"2023-07-14T11:10:14.357605+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:06:23.046904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737806+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:10:07.737806+0000","last_peered":"2023-07-14T11:10:07.737806+0000","last_clean":"2023-07-14T11:10:07.737806+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:10:07.737806+0000","last_undegraded":"2023-07-14T11:10:07.737806+0000","last_fullsized":"2023-07-14T11:10:07.737806+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:35:15.765387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.370106+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:10:15.370106+0000","last_peered":"2023-07-14T11:10:15.370106+0000","last_clean":"2023-07-14T11:10:15.370106+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:10:15.370106+0000","last_undegraded":"2023-07-14T11:10:15.370106+0000","last_fullsized":"2023-07-14T11:10:15.370106+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:27:03.761811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734605+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:10:07.734605+0000","last_peered":"2023-07-14T11:10:07.734605+0000","last_clean":"2023-07-14T11:10:07.734605+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:10:07.734605+0000","last_undegraded":"2023-07-14T11:10:07.734605+0000","last_fullsized":"2023-07-14T11:10:07.734605+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:17:38.486109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738707+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:10:07.738707+0000","last_peered":"2023-07-14T11:10:07.738707+0000","last_clean":"2023-07-14T11:10:07.738707+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:10:07.738707+0000","last_undegraded":"2023-07-14T11:10:07.738707+0000","last_fullsized":"2023-07-14T11:10:07.738707+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:57.136733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"90'7","reported_seq":43,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.369508+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:10:15.369508+0000","last_peered":"2023-07-14T11:10:15.369508+0000","last_clean":"2023-07-14T11:10:15.369508+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:10:15.369508+0000","last_undegraded":"2023-07-14T11:10:15.369508+0000","last_fullsized":"2023-07-14T11:10:15.369508+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:47:01.844602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740024+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:10:07.740024+0000","last_peered":"2023-07-14T11:10:07.740024+0000","last_clean":"2023-07-14T11:10:07.740024+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:10:07.740024+0000","last_undegraded":"2023-07-14T11:10:07.740024+0000","last_fullsized":"2023-07-14T11:10:07.740024+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:34:33.255530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738545+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:10:07.738545+0000","last_peered":"2023-07-14T11:10:07.738545+0000","last_clean":"2023-07-14T11:10:07.738545+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:10:07.738545+0000","last_undegraded":"2023-07-14T11:10:07.738545+0000","last_fullsized":"2023-07-14T11:10:07.738545+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:36.240727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375648+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:10:16.375648+0000","last_peered":"2023-07-14T11:10:16.375648+0000","last_clean":"2023-07-14T11:10:16.375648+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:10:16.375648+0000","last_undegraded":"2023-07-14T11:10:16.375648+0000","last_fullsized":"2023-07-14T11:10:16.375648+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:02:18.139490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141864+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:10:08.141864+0000","last_peered":"2023-07-14T11:10:08.141864+0000","last_clean":"2023-07-14T11:10:08.141864+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:10:08.141864+0000","last_undegraded":"2023-07-14T11:10:08.141864+0000","last_fullsized":"2023-07-14T11:10:08.141864+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:28:33.937520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823662+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:10:07.823662+0000","last_peered":"2023-07-14T11:10:07.823662+0000","last_clean":"2023-07-14T11:10:07.823662+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:10:07.823662+0000","last_undegraded":"2023-07-14T11:10:07.823662+0000","last_fullsized":"2023-07-14T11:10:07.823662+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:28:39.306429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":34,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.782018+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:10:13.782018+0000","last_peered":"2023-07-14T11:10:13.782018+0000","last_clean":"2023-07-14T11:10:13.782018+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:10:13.782018+0000","last_undegraded":"2023-07-14T11:10:13.782018+0000","last_fullsized":"2023-07-14T11:10:13.782018+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:50:14.638370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734307+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:10:07.734307+0000","last_peered":"2023-07-14T11:10:07.734307+0000","last_clean":"2023-07-14T11:10:07.734307+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:10:07.734307+0000","last_undegraded":"2023-07-14T11:10:07.734307+0000","last_fullsized":"2023-07-14T11:10:07.734307+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:27:40.728690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740731+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:10:07.740731+0000","last_peered":"2023-07-14T11:10:07.740731+0000","last_clean":"2023-07-14T11:10:07.740731+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:10:07.740731+0000","last_undegraded":"2023-07-14T11:10:07.740731+0000","last_fullsized":"2023-07-14T11:10:07.740731+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:04:37.925348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"90'7","reported_seq":33,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375880+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:10:16.375880+0000","last_peered":"2023-07-14T11:10:16.375880+0000","last_clean":"2023-07-14T11:10:16.375880+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:10:16.375880+0000","last_undegraded":"2023-07-14T11:10:16.375880+0000","last_fullsized":"2023-07-14T11:10:16.375880+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:35:40.777893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737586+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:10:07.737586+0000","last_peered":"2023-07-14T11:10:07.737586+0000","last_clean":"2023-07-14T11:10:07.737586+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:10:07.737586+0000","last_undegraded":"2023-07-14T11:10:07.737586+0000","last_fullsized":"2023-07-14T11:10:07.737586+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:43.286237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823509+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:10:07.823509+0000","last_peered":"2023-07-14T11:10:07.823509+0000","last_clean":"2023-07-14T11:10:07.823509+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:10:07.823509+0000","last_undegraded":"2023-07-14T11:10:07.823509+0000","last_fullsized":"2023-07-14T11:10:07.823509+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:28:35.378022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"90'3","reported_seq":29,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.843598+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:10:15.843598+0000","last_peered":"2023-07-14T11:10:15.843598+0000","last_clean":"2023-07-14T11:10:15.843598+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:10:15.843598+0000","last_undegraded":"2023-07-14T11:10:15.843598+0000","last_fullsized":"2023-07-14T11:10:15.843598+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:10.923637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739938+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:10:07.739938+0000","last_peered":"2023-07-14T11:10:07.739938+0000","last_clean":"2023-07-14T11:10:07.739938+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:10:07.739938+0000","last_undegraded":"2023-07-14T11:10:07.739938+0000","last_fullsized":"2023-07-14T11:10:07.739938+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:27:36.946955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823764+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:10:07.823764+0000","last_peered":"2023-07-14T11:10:07.823764+0000","last_clean":"2023-07-14T11:10:07.823764+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:10:07.823764+0000","last_undegraded":"2023-07-14T11:10:07.823764+0000","last_fullsized":"2023-07-14T11:10:07.823764+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:46:25.967420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":31,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:13.783411+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:10:13.783411+0000","last_peered":"2023-07-14T11:10:13.783411+0000","last_clean":"2023-07-14T11:10:13.783411+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:10:13.783411+0000","last_undegraded":"2023-07-14T11:10:13.783411+0000","last_fullsized":"2023-07-14T11:10:13.783411+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:08:54.674537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736919+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:10:07.736919+0000","last_peered":"2023-07-14T11:10:07.736919+0000","last_clean":"2023-07-14T11:10:07.736919+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:10:07.736919+0000","last_undegraded":"2023-07-14T11:10:07.736919+0000","last_fullsized":"2023-07-14T11:10:07.736919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:34:43.143864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734396+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:10:07.734396+0000","last_peered":"2023-07-14T11:10:07.734396+0000","last_clean":"2023-07-14T11:10:07.734396+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:10:07.734396+0000","last_undegraded":"2023-07-14T11:10:07.734396+0000","last_fullsized":"2023-07-14T11:10:07.734396+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:13:42.740364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"90'5","reported_seq":78,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:11:00.487390+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:11:00.487390+0000","last_peered":"2023-07-14T11:11:00.487390+0000","last_clean":"2023-07-14T11:11:00.487390+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:11:00.487390+0000","last_undegraded":"2023-07-14T11:11:00.487390+0000","last_fullsized":"2023-07-14T11:11:00.487390+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:17:09.808452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1122,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":47,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737674+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:10:07.737674+0000","last_peered":"2023-07-14T11:10:07.737674+0000","last_clean":"2023-07-14T11:10:07.737674+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:10:07.737674+0000","last_undegraded":"2023-07-14T11:10:07.737674+0000","last_fullsized":"2023-07-14T11:10:07.737674+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:51:08.137181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141293+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:10:08.141293+0000","last_peered":"2023-07-14T11:10:08.141293+0000","last_clean":"2023-07-14T11:10:08.141293+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:10:08.141293+0000","last_undegraded":"2023-07-14T11:10:08.141293+0000","last_fullsized":"2023-07-14T11:10:08.141293+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:33:53.668586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"90'3","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:15.366487+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:10:15.366487+0000","last_peered":"2023-07-14T11:10:15.366487+0000","last_clean":"2023-07-14T11:10:15.366487+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:10:15.366487+0000","last_undegraded":"2023-07-14T11:10:15.366487+0000","last_fullsized":"2023-07-14T11:10:15.366487+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:41:36.321965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.736750+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:10:07.736750+0000","last_peered":"2023-07-14T11:10:07.736750+0000","last_clean":"2023-07-14T11:10:07.736750+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:10:07.736750+0000","last_undegraded":"2023-07-14T11:10:07.736750+0000","last_fullsized":"2023-07-14T11:10:07.736750+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:11:06.323972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":17,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739174+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:10:07.739174+0000","last_peered":"2023-07-14T11:10:07.739174+0000","last_clean":"2023-07-14T11:10:07.739174+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:10:07.739174+0000","last_undegraded":"2023-07-14T11:10:07.739174+0000","last_fullsized":"2023-07-14T11:10:07.739174+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:14:16.916714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"90'5","reported_seq":41,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:16.375147+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:10:16.375147+0000","last_peered":"2023-07-14T11:10:16.375147+0000","last_clean":"2023-07-14T11:10:16.375147+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:10:16.375147+0000","last_undegraded":"2023-07-14T11:10:16.375147+0000","last_fullsized":"2023-07-14T11:10:16.375147+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:33.874908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737919+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:10:07.737919+0000","last_peered":"2023-07-14T11:10:07.737919+0000","last_clean":"2023-07-14T11:10:07.737919+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:10:07.737919+0000","last_undegraded":"2023-07-14T11:10:07.737919+0000","last_fullsized":"2023-07-14T11:10:07.737919+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:33:56.102535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737624+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:10:07.737624+0000","last_peered":"2023-07-14T11:10:07.737624+0000","last_clean":"2023-07-14T11:10:07.737624+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:10:07.737624+0000","last_undegraded":"2023-07-14T11:10:07.737624+0000","last_fullsized":"2023-07-14T11:10:07.737624+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:49.125150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738992+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:10:07.738992+0000","last_peered":"2023-07-14T11:10:07.738992+0000","last_clean":"2023-07-14T11:10:07.738992+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:10:07.738992+0000","last_undegraded":"2023-07-14T11:10:07.738992+0000","last_fullsized":"2023-07-14T11:10:07.738992+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:49:40.242321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.738280+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:10:07.738280+0000","last_peered":"2023-07-14T11:10:07.738280+0000","last_clean":"2023-07-14T11:10:07.738280+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:10:07.738280+0000","last_undegraded":"2023-07-14T11:10:07.738280+0000","last_fullsized":"2023-07-14T11:10:07.738280+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:00:40.108545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.141672+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:10:08.141672+0000","last_peered":"2023-07-14T11:10:08.141672+0000","last_clean":"2023-07-14T11:10:08.141672+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:10:08.141672+0000","last_undegraded":"2023-07-14T11:10:08.141672+0000","last_fullsized":"2023-07-14T11:10:08.141672+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:35:25.035847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:08.143146+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:10:08.143146+0000","last_peered":"2023-07-14T11:10:08.143146+0000","last_clean":"2023-07-14T11:10:08.143146+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:10:08.143146+0000","last_undegraded":"2023-07-14T11:10:08.143146+0000","last_fullsized":"2023-07-14T11:10:08.143146+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:00:48.749245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":30,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.744121+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:10:07.744121+0000","last_peered":"2023-07-14T11:10:07.744121+0000","last_clean":"2023-07-14T11:10:07.744121+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:10:07.744121+0000","last_undegraded":"2023-07-14T11:10:07.744121+0000","last_fullsized":"2023-07-14T11:10:07.744121+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:56:30.382410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.740340+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:10:07.740340+0000","last_peered":"2023-07-14T11:10:07.740340+0000","last_clean":"2023-07-14T11:10:07.740340+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:10:07.740340+0000","last_undegraded":"2023-07-14T11:10:07.740340+0000","last_fullsized":"2023-07-14T11:10:07.740340+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:27.366892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":11,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.823702+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:10:07.823702+0000","last_peered":"2023-07-14T11:10:07.823702+0000","last_clean":"2023-07-14T11:10:07.823702+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:10:07.823702+0000","last_undegraded":"2023-07-14T11:10:07.823702+0000","last_fullsized":"2023-07-14T11:10:07.823702+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:42.506429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739599+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:10:07.739599+0000","last_peered":"2023-07-14T11:10:07.739599+0000","last_clean":"2023-07-14T11:10:07.739599+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:10:07.739599+0000","last_undegraded":"2023-07-14T11:10:07.739599+0000","last_fullsized":"2023-07-14T11:10:07.739599+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:09:31.536272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.739315+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:10:07.739315+0000","last_peered":"2023-07-14T11:10:07.739315+0000","last_clean":"2023-07-14T11:10:07.739315+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:10:07.739315+0000","last_undegraded":"2023-07-14T11:10:07.739315+0000","last_fullsized":"2023-07-14T11:10:07.739315+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:06:45.531716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734952+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:10:07.734952+0000","last_peered":"2023-07-14T11:10:07.734952+0000","last_clean":"2023-07-14T11:10:07.734952+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:10:07.734952+0000","last_undegraded":"2023-07-14T11:10:07.734952+0000","last_fullsized":"2023-07-14T11:10:07.734952+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:37:01.416822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.734822+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:10:07.734822+0000","last_peered":"2023-07-14T11:10:07.734822+0000","last_clean":"2023-07-14T11:10:07.734822+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:10:07.734822+0000","last_undegraded":"2023-07-14T11:10:07.734822+0000","last_fullsized":"2023-07-14T11:10:07.734822+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:02:57.300787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":24,"reported_epoch":90,"state":"active+clean","last_fresh":"2023-07-14T11:10:07.737927+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:10:07.737927+0000","last_peered":"2023-07-14T11:10:07.737927+0000","last_clean":"2023-07-14T11:10:07.737927+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:10:07.737927+0000","last_undegraded":"2023-07-14T11:10:07.737927+0000","last_fullsized":"2023-07-14T11:10:07.737927+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:53:58.972191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":10024,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":481,"num_read_kb":257,"num_write":209,"num_write_kb":67,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":30072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183,"ondisk_log_size":183,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416413,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27940,"kb_used_data":988,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717212,"statfs":{"total":95995035648,"available":95966425088,"internally_reserved":0,"allocated":1011712,"data_stored":489881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.59299999999999997}]},{"osd":1,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.72199999999999998}]},{"osd":2,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.626}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.77900000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.84599999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.89700000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.86699999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.94299999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6950000000000001,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.96299999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:09:57 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.99299999999999999}]}]},{"osd":10,"up_from":72,"seq":309237645343,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28564,"kb_used_data":1612,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716588,"statfs":{"total":95995035648,"available":95965786112,"internally_reserved":0,"allocated":1650688,"data_stored":1084587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":3.9740000000000002,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":4.0190000000000001,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.90500000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.46300000000000002,"15min":0.45700000000000002},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.9419999999999999,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.7210000000000001,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.57299999999999995}]},{"osd":2,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.64500000000000002,"15min":0.64000000000000001},"min":{"1min":0.47499999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0880000000000001,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.67200000000000004,"15min":0.67000000000000004},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.796,"5min":1.796,"15min":1.796},"last":0.70199999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.876,"15min":0.90400000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9430000000000001,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.67300000000000004,"15min":0.67600000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.755}]},{"osd":4,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.56999999999999995,"15min":0.56200000000000006},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":2.0499999999999998,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.72499999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.72399999999999998,"15min":0.72399999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.014,"5min":1.165,"15min":1.165},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.76900000000000002,"15min":0.77100000000000002},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.9850000000000001,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78100000000000003,"15min":0.78200000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.897,"5min":1.897,"15min":1.897},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.91800000000000004,"15min":0.94399999999999995},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.5129999999999999,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.96499999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.78100000000000003,"15min":0.78200000000000003},"min":{"1min":0.54800000000000004,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.107,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.84199999999999997,"15min":0.85199999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.052,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.94999999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.84999999999999998,"15min":0.84799999999999998},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":2.1970000000000001,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":1.0169999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.88300000000000001,"15min":0.89100000000000001},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.6870000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.98799999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:10:00 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":2.1440000000000001,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.89200000000000002},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":4.8609999999999998,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":1.048}]}]},{"osd":9,"up_from":67,"seq":287762808868,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":1012,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":1036288,"data_stored":489859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.51800000000000002,"15min":0.53100000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.59199999999999997,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.61199999999999999,"15min":0.628},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.94099999999999995,"5min":1.488,"15min":1.488},"last":0.54100000000000004}]},{"osd":1,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.76100000000000001,"15min":0.76900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.69799999999999995,"15min":0.70699999999999996},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.89600000000000002,"5min":1.0609999999999999,"15min":1.0609999999999999},"last":0.68700000000000006}]},{"osd":2,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.74199999999999999,"15min":0.76000000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.89900000000000002,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.432},{"interface":"front","average":{"1min":0.62,"5min":0.73199999999999998,"15min":0.751},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.78900000000000003,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.49399999999999999}]},{"osd":3,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.71099999999999997,"15min":0.73099999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.0700000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.69899999999999995,"15min":0.71899999999999997},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81499999999999995,"5min":1.292,"15min":1.292},"last":0.47299999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.75700000000000001,"15min":0.75700000000000001},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.80900000000000005,"15min":0.81699999999999995},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.0529999999999999,"5min":1.165,"15min":1.165},"last":0.83999999999999997}]},{"osd":5,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.84799999999999998,"15min":0.86899999999999999},"min":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.006,"5min":1.242,"15min":1.242},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.81799999999999995,"15min":0.84099999999999997},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.93700000000000006,"5min":1.258,"15min":1.258},"last":0.51800000000000002}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.93700000000000006,"15min":0.94899999999999995},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":1.8069999999999999,"5min":1.8069999999999999,"15min":1.8069999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.95699999999999996,"15min":0.97699999999999998},"min":{"1min":0.65100000000000002,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.7190000000000001,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.621}]},{"osd":8,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.89200000000000002,"15min":0.91200000000000003},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":0.99199999999999999,"5min":1.331,"15min":1.331},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.97999999999999998,"15min":1.002},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.1240000000000001,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.78000000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.91000000000000003,"15min":0.91600000000000004},"min":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"max":{"1min":1.7310000000000001,"5min":1.7310000000000001,"15min":1.7310000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.96999999999999997,"15min":0.97599999999999998},"min":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"max":{"1min":1.7010000000000001,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":0.754}]},{"osd":11,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.335,"5min":1.335,"15min":1.335},"last":0.64000000000000001}]}]},{"osd":8,"up_from":59,"seq":253403070505,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27960,"kb_used_data":1008,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717192,"statfs":{"total":95995035648,"available":95966404608,"internally_reserved":0,"allocated":1032192,"data_stored":490237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83899999999999997,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59399999999999997,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.84899999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.58199999999999996}]},{"osd":1,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.55900000000000005,"15min":0.53600000000000003},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78200000000000003,"5min":0.875,"15min":0.875},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.61499999999999999,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79800000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.56200000000000006}]},{"osd":2,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.69599999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72299999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60699999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.40300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93000000000000005,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91800000000000004,"5min":1.079,"15min":1.079},"last":0.73199999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.61899999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.89700000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.64800000000000002,"15min":0.63600000000000001},"min":{"1min":0.56799999999999995,"5min":0.37,"15min":0.37},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.79500000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.79800000000000004,"5min":1.052,"15min":1.052},"last":0.75},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82699999999999996,"5min":1.135,"15min":1.135},"last":0.84899999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.70599999999999996,"15min":0.70799999999999996},"min":{"1min":0.48899999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90100000000000002,"5min":1.008,"15min":1.008},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.73599999999999999,"15min":0.74199999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.96099999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.71699999999999997},"min":{"1min":0.58199999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":1.028,"15min":1.028},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.73799999999999999,"15min":0.73199999999999998},"min":{"1min":0.54500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.99399999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.89100000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.749,"15min":0.751},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.77700000000000002,"15min":0.77400000000000002},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.92400000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.73199999999999998,"15min":0.71999999999999997},"min":{"1min":0.53400000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.75900000000000001,"15min":0.74299999999999999},"min":{"1min":0.65900000000000003,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.97399999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.90500000000000003}]}]},{"osd":7,"up_from":51,"seq":219043332141,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28536,"kb_used_data":1584,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716616,"statfs":{"total":95995035648,"available":95965814784,"internally_reserved":0,"allocated":1622016,"data_stored":1081830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.318,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60399999999999998,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.34399999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.64200000000000002,"15min":0.65200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.91700000000000004,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.625,"15min":0.623},"min":{"1min":0.503,"5min":0.435,"15min":0.435},"max":{"1min":0.85899999999999999,"5min":0.872,"15min":0.872},"last":0.63800000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.72099999999999997,"5min":0.61299999999999999,"15min":0.58699999999999997},"min":{"1min":0.42499999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":3.6520000000000001,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.627,"15min":0.59999999999999998},"min":{"1min":0.42299999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.7690000000000001,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.55900000000000005,"15min":0.54900000000000004},"min":{"1min":0.371,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77600000000000002,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58099999999999996,"15min":0.57999999999999996},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.88700000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.41099999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.71399999999999997,"15min":0.72499999999999998},"min":{"1min":0.48999999999999999,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.84499999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70899999999999996,"15min":0.71499999999999997},"min":{"1min":0.502,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.75}]},{"osd":6,"last update":"Fri Jul 14 11:10:29 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.749,"15min":0.73999999999999999},"min":{"1min":0.45900000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.79600000000000004,"15min":0.74399999999999999},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":7.3719999999999999,"5min":7.3719999999999999,"15min":7.3719999999999999},"last":0.48899999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.76100000000000001,"15min":0.75900000000000001},"min":{"1min":0.65200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.96599999999999997,"5min":1,"15min":1},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72199999999999998,"15min":0.71999999999999997},"min":{"1min":0.625,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.76900000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":0.80700000000000005,"15min":0.80800000000000005},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0169999999999999,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79700000000000004,"15min":0.79300000000000004},"min":{"1min":0.58699999999999997,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.107,"5min":1.1599999999999999,"15min":1.1599999999999999},"last":0.58999999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:10:36 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.72299999999999998,"15min":0.71799999999999997},"min":{"1min":0.53800000000000003,"5min":0.442,"15min":0.442},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.79200000000000004,"15min":0.79100000000000004},"min":{"1min":0.58499999999999996,"5min":0.37,"15min":0.37},"max":{"1min":1.01,"5min":1.216,"15min":1.216},"last":0.71099999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"min":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"min":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.82199999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495666,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":992,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":1015808,"data_stored":489411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48799999999999999,"15min":0.46999999999999997},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.66700000000000004,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.628,"15min":0.60999999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.95899999999999996,"5min":1.151,"15min":1.151},"last":0.58699999999999997}]},{"osd":1,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.67900000000000005,"15min":0.67500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90200000000000002,"5min":1.407,"15min":1.407},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.56599999999999995}]},{"osd":2,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.74199999999999999,"15min":0.72899999999999998},"min":{"1min":0.60299999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":0.995,"5min":1.087,"15min":1.087},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.70399999999999996,"15min":0.70299999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.0760000000000001,"5min":1.0760000000000001,"15min":1.0760000000000001},"last":0.91300000000000003}]},{"osd":3,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.64000000000000001,"15min":0.63200000000000001},"min":{"1min":0.40999999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.89000000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.60299999999999998,"15min":0.59699999999999998},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.747,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.67600000000000005}]},{"osd":4,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":0.76000000000000001,"15min":0.745},"min":{"1min":0.505,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.091,"5min":1.173,"15min":1.173},"last":0.438},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0089999999999999,"5min":1.302,"15min":1.302},"last":0.47799999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:08 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.72399999999999998,"15min":0.73899999999999999},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86599999999999999,"5min":1.004,"15min":1.004},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.82699999999999996,"15min":0.81899999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":1.1890000000000001,"5min":1.264,"15min":1.264},"last":0.65500000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.83299999999999996,"15min":0.80500000000000005},"min":{"1min":0.59599999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2549999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.82899999999999996,"15min":0.81699999999999995},"min":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":1.157,"5min":1.502,"15min":1.502},"last":0.622}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.64800000000000002,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.93400000000000005,"15min":0.96599999999999997},"min":{"1min":0.62,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2270000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.84999999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.002,"15min":1.0209999999999999},"min":{"1min":0.64400000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2849999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.038},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.0209999999999999,"15min":1.04},"min":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"max":{"1min":1.409,"5min":1.466,"15min":1.466},"last":0.87}]},{"osd":10,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.98199999999999998,"15min":0.98499999999999999},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.304,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.86899999999999999,"15min":0.874},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.3140000000000001,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.83099999999999996}]},{"osd":11,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":0.996},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.88400000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574923,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27952,"kb_used_data":1000,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717200,"statfs":{"total":95995035648,"available":95966412800,"internally_reserved":0,"allocated":1024000,"data_stored":494235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:22 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.42199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56799999999999995,"15min":0.52000000000000002},"min":{"1min":0.41999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.81999999999999995,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.65200000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:10:37 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.625,"15min":0.55700000000000005},"min":{"1min":0.43099999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94699999999999995,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.57399999999999995,"15min":0.52200000000000002},"min":{"1min":0.45100000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.85699999999999998,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.82699999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:09:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.56999999999999995,"15min":0.55200000000000005},"min":{"1min":0.38400000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77700000000000002,"5min":1.5269999999999999,"15min":1.5269999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.64600000000000002,"15min":0.58499999999999996},"min":{"1min":0.57099999999999995,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.3819999999999997,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.77200000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.68500000000000005,"15min":0.65000000000000002},"min":{"1min":0.497,"5min":0.38,"15min":0.38},"max":{"1min":0.95499999999999996,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.63300000000000001,"15min":0.60299999999999998},"min":{"1min":0.48999999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.90400000000000003,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.89600000000000002}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.68000000000000005,"15min":0.66300000000000003},"min":{"1min":0.53600000000000003,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86799999999999999,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.72999999999999998},"min":{"1min":0.49399999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.1180000000000001,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":0.97499999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:10:27 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.70899999999999996,"15min":0.67300000000000004},"min":{"1min":0.45000000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.2110000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.64400000000000002,"15min":0.624},"min":{"1min":0.59699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97499999999999998,"5min":0.97499999999999998,"15min":0.97499999999999998},"last":1.1020000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.68999999999999995,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.01,"5min":1.04,"15min":1.04},"last":0.999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.77800000000000002,"15min":0.72999999999999998},"min":{"1min":0.62,"5min":0.441,"15min":0.441},"max":{"1min":1.159,"5min":1.165,"15min":1.165},"last":1.1200000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:10:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":0.754,"15min":0.70399999999999996},"min":{"1min":0.55000000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":5.4260000000000002,"5min":5.4260000000000002,"15min":5.4260000000000002},"last":1.012},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.752,"15min":0.73399999999999999},"min":{"1min":0.67600000000000005,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":1.0369999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.85099999999999998,"15min":0.83699999999999997},"min":{"1min":0.57499999999999996,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.056},{"interface":"front","average":{"1min":0.82099999999999995,"5min":0.73499999999999999,"15min":0.72099999999999997},"min":{"1min":0.58799999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":1.0720000000000001}]},{"osd":11,"last update":"Fri Jul 14 11:10:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"min":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.157},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"min":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":1.4179999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738447,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27964,"kb_used_data":1012,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717188,"statfs":{"total":95995035648,"available":95966400512,"internally_reserved":0,"allocated":1036288,"data_stored":490594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:10:16 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.61899999999999999,"15min":0.56299999999999994},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.94499999999999995,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.629,"15min":0.56699999999999995},"min":{"1min":0.46400000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.86699999999999999,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.48999999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.64100000000000001,"15min":0.57799999999999996},"min":{"1min":0.52400000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1899999999999999,"5min":2.528,"15min":2.528},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.64700000000000002,"15min":0.59899999999999998},"min":{"1min":0.377,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.1280000000000001,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.79500000000000004}]},{"osd":3,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":3.2599999999999998,"5min":3.2599999999999998,"15min":3.2599999999999998},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.1500000000000004,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.50900000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:10:25 2023","interfaces":[{"interface":"back","average":{"1min":0.74399999999999999,"5min":0.67300000000000004,"15min":0.625},"min":{"1min":0.48399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.1120000000000001,"5min":2.48,"15min":2.48},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72899999999999998,"15min":0.68899999999999995},"min":{"1min":0.45200000000000001,"5min":0.443,"15min":0.443},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.72199999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":0.78200000000000003,"15min":0.70299999999999996},"min":{"1min":0.54000000000000004,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.0839999999999996,"5min":6.0839999999999996,"15min":6.0839999999999996},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.71699999999999997,"15min":0.65700000000000003},"min":{"1min":0.50900000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.246,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":1.02}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.70599999999999996,"15min":0.69599999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.99199999999999999,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.71199999999999997,"15min":0.70799999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.034,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":0.42699999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:10:46 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.323,"5min":1.323,"15min":1.323},"last":0.93799999999999994},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.9239999999999999,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.749}]},{"osd":8,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.91100000000000003,"15min":0.88200000000000001},"min":{"1min":0.55700000000000005,"5min":0.496,"15min":0.496},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.88100000000000001,"15min":0.89100000000000001},"min":{"1min":0.68600000000000005,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.91100000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:10:21 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.79700000000000004,"15min":0.79500000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.1000000000000001,"5min":1.232,"15min":1.232},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":0.89800000000000002,"15min":0.90000000000000002},"min":{"1min":0.49399999999999999,"5min":0.437,"15min":0.437},"max":{"1min":1.2050000000000001,"5min":1.587,"15min":1.587},"last":0.86799999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:10:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.879,"15min":0.86799999999999999},"min":{"1min":0.57999999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":2.085,"5min":2.085,"15min":2.085},"last":0.62},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.83899999999999997,"15min":0.82099999999999995},"min":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.97799999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.95499999999999996,"15min":0.90000000000000002},"min":{"1min":0.77200000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":6.0460000000000003,"5min":6.0460000000000003,"15min":6.0460000000000003},"last":0.95199999999999996},{"interface":"front","average":{"1min":1.026,"5min":0.88800000000000001,"15min":0.86499999999999999},"min":{"1min":0.752,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.84899999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411398,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27968,"kb_used_data":1016,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717184,"statfs":{"total":95995035648,"available":95966396416,"internally_reserved":0,"allocated":1040384,"data_stored":494134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.54800000000000004,"15min":0.54200000000000004},"min":{"1min":0.41299999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.304,"5min":1.304,"15min":1.304},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56100000000000005,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.505}]},{"osd":1,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.56499999999999995,"15min":0.53900000000000003},"min":{"1min":0.42299999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.58899999999999997,"15min":0.55600000000000005},"min":{"1min":0.45300000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.76800000000000002}]},{"osd":3,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.61899999999999999,"15min":0.60499999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.70599999999999996,"15min":0.64700000000000002},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":3.2210000000000001,"5min":3.2210000000000001,"15min":3.2210000000000001},"last":0.70099999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"min":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.746,"5min":0.746,"15min":0.746},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.79500000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.48599999999999999,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73899999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.501,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.84099999999999997,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65500000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:10:09 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.60899999999999999},"min":{"1min":0.48399999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.66200000000000003,"15min":0.63800000000000001},"min":{"1min":0.47299999999999998,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.94699999999999995,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.749}]},{"osd":7,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.86699999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.60399999999999998,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.61499999999999999,"15min":0.58899999999999997},"min":{"1min":0.51800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.89000000000000001}]},{"osd":9,"last update":"Fri Jul 14 11:10:15 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.745,"15min":0.74299999999999999},"min":{"1min":0.63100000000000001,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":0.93300000000000005,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.80200000000000005,"15min":0.80000000000000004},"min":{"1min":0.60099999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.85299999999999998}]},{"osd":10,"last update":"Fri Jul 14 11:10:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.78600000000000003,"15min":0.77800000000000002},"min":{"1min":0.52200000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.878},{"interface":"front","average":{"1min":0.80900000000000005,"5min":0.745,"15min":0.73499999999999999},"min":{"1min":0.55700000000000005,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.92600000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.95199999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182463,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27952,"kb_used_data":1000,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717200,"statfs":{"total":95995035648,"available":95966412800,"internally_reserved":0,"allocated":1024000,"data_stored":488894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.45200000000000001,"15min":0.40999999999999998},"min":{"1min":0.33800000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.66200000000000003,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.47599999999999998,"15min":0.436},"min":{"1min":0.36099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.81299999999999994,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.53500000000000003}]},{"osd":1,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.38100000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":1.591},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.65800000000000003,"15min":0.59699999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89700000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.83899999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65800000000000003,"15min":0.63300000000000001},"min":{"1min":0.504,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94099999999999995,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.67400000000000004,"15min":0.64600000000000002},"min":{"1min":0.504,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.046,"5min":1.069,"15min":1.069},"last":0.624}]},{"osd":4,"last update":"Fri Jul 14 11:10:24 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.68300000000000005,"15min":0.63800000000000001},"min":{"1min":0.46600000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":1.742},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.746,"15min":0.69199999999999995},"min":{"1min":0.48299999999999998,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.98199999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.75800000000000001,"15min":0.73899999999999999},"min":{"1min":0.54400000000000004,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.022,"5min":1.262,"15min":1.262},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.75900000000000001,"15min":0.752},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.1479999999999999,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.68000000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:10:14 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.752,"15min":0.72999999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.1739999999999999,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":1.044},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.752,"15min":0.76000000000000001},"min":{"1min":0.501,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.0549999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.1859999999999999}]},{"osd":7,"last update":"Fri Jul 14 11:10:33 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.81799999999999995,"15min":0.82799999999999996},"min":{"1min":0.64600000000000002,"5min":0.496,"15min":0.496},"max":{"1min":1.012,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":1.2649999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.81699999999999995,"15min":0.81100000000000005},"min":{"1min":0.63500000000000001,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.097,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":1.7050000000000001}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:10:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.90000000000000002,"15min":0.91100000000000003},"min":{"1min":0.58699999999999997,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.657},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.98599999999999999,"15min":0.996},"min":{"1min":0.64700000000000002,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.401,"5min":1.504,"15min":1.504},"last":1.075}]},{"osd":10,"last update":"Fri Jul 14 11:10:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.0369999999999999,"15min":1.036},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":4.0300000000000002,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":1.23},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.96599999999999997,"15min":0.97399999999999998},"min":{"1min":0.72199999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.1359999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.94699999999999995}]},{"osd":11,"last update":"Fri Jul 14 11:10:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"min":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"max":{"1min":1.351,"5min":1.351,"15min":1.351},"last":1.296},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":1.014}]}]},{"osd":4,"up_from":33,"seq":141733920829,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27944,"kb_used_data":992,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717208,"statfs":{"total":95995035648,"available":95966420992,"internally_reserved":0,"allocated":1015808,"data_stored":489116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38400000000000001,"15min":0.36599999999999999},"min":{"1min":0.28799999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.56799999999999995,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40999999999999998,"15min":0.38800000000000001},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61099999999999999,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.29299999999999998}]},{"osd":2,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.59999999999999998,"15min":0.57999999999999996},"min":{"1min":0.36799999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.624,"15min":0.58299999999999996},"min":{"1min":0.51200000000000001,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.69199999999999995}]},{"osd":3,"last update":"Fri Jul 14 11:10:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.61099999999999999,"15min":0.58399999999999996},"min":{"1min":0.441,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.94999999999999996,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.60199999999999998,"15min":0.58599999999999997},"min":{"1min":0.45100000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.878,"5min":1.054,"15min":1.054},"last":0.60599999999999998}]},{"osd":5,"last update":"Fri Jul 14 11:10:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.61399999999999999,"15min":0.58599999999999997},"min":{"1min":0.44800000000000001,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.89400000000000002,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66500000000000004,"15min":0.63600000000000001},"min":{"1min":0.46800000000000003,"5min":0.373,"15min":0.373},"max":{"1min":1.0209999999999999,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.74299999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:10:11 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.65100000000000002,"15min":0.63500000000000001},"min":{"1min":0.44500000000000001,"5min":0.432,"15min":0.432},"max":{"1min":0.83499999999999996,"5min":1.105,"15min":1.105},"last":0.436},{"interface":"front","average":{"1min":0.753,"5min":0.71299999999999997,"15min":0.68100000000000005},"min":{"1min":0.51000000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.90400000000000003,"5min":1.137,"15min":1.137},"last":0.63}]},{"osd":7,"last update":"Fri Jul 14 11:10:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.64900000000000002,"15min":0.61699999999999999},"min":{"1min":0.40400000000000003,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.996,"5min":1.087,"15min":1.087},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.623,"15min":0.59599999999999997},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.47799999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.73199999999999998,"15min":0.72799999999999998},"min":{"1min":0.54900000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.81000000000000005,"15min":0.80100000000000005},"min":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0329999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.78100000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:10:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":0.83899999999999997,"15min":0.84899999999999998},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.81699999999999995,"15min":0.82499999999999996},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.123,"5min":1.123,"15min":1.123},"last":0.67100000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:10:35 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.88200000000000001,"15min":0.88800000000000001},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.1659999999999999,"5min":1.212,"15min":1.212},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.78800000000000003,"5min":0.80200000000000005,"15min":0.80400000000000005},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.0569999999999999,"5min":1.073,"15min":1.073},"last":0.51100000000000001}]},{"osd":11,"last update":"Fri Jul 14 11:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.65200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757304,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28548,"kb_used_data":1596,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716604,"statfs":{"total":95995035648,"available":95965802496,"internally_reserved":0,"allocated":1634304,"data_stored":1081604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.55100000000000005,"15min":0.56100000000000005},"min":{"1min":0.34999999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.77600000000000002,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.40400000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.78700000000000003,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.61299999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.45800000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73999999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.59999999999999998,"15min":0.58999999999999997},"min":{"1min":0.47899999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82799999999999996,"5min":1.466,"15min":1.466},"last":0.73399999999999999}]},{"osd":2,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.437,"15min":0.42799999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.442,"15min":0.435},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.55700000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.58799999999999997,"15min":0.59799999999999998},"min":{"1min":0.42399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.79100000000000004,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61399999999999999,"15min":0.61499999999999999},"min":{"1min":0.46300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77300000000000002,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.754}]},{"osd":4,"last update":"Fri Jul 14 11:10:48 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.70199999999999996,"15min":0.65500000000000003},"min":{"1min":0.51900000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.79600000000000004,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.629,"5min":0.68000000000000005,"15min":0.64000000000000001},"min":{"1min":0.48299999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.81100000000000005,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.65700000000000003}]},{"osd":6,"last update":"Fri Jul 14 11:10:07 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.65800000000000003,"15min":0.66400000000000003},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88400000000000001,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68300000000000005,"15min":0.68899999999999995},"min":{"1min":0.51500000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.82399999999999995,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.84099999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:10:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70199999999999996,"15min":0.70799999999999996},"min":{"1min":0.55600000000000005,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.86499999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.73599999999999999,"15min":0.73999999999999999},"min":{"1min":0.59499999999999997,"5min":0.496,"15min":0.496},"max":{"1min":1.0489999999999999,"5min":1.528,"15min":1.528},"last":0.86799999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.878,"15min":0.878},"last":0.77700000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:10:13 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.746,"15min":0.74099999999999999},"min":{"1min":0.58799999999999997,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.93000000000000005,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.76100000000000001,"15min":0.75700000000000001},"min":{"1min":0.57499999999999996,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.88,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.94399999999999995}]},{"osd":10,"last update":"Fri Jul 14 11:10:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.73799999999999999,"15min":0.72999999999999998},"min":{"1min":0.65300000000000002,"5min":0.439,"15min":0.439},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":0.95099999999999996,"15min":0.97599999999999998},"min":{"1min":0.66200000000000003,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.97299999999999998,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.90800000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":2772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":2998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:11:04.340 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-07-14T11:11:04.340 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-07-14T11:11:04.341 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-07-14T11:11:04.341 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph health --format=json 2023-07-14T11:11:05.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:04 smithi130 ceph-mon[129405]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:05.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:04 smithi040 ceph-mon[97954]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:05.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:04 smithi045 ceph-mon[128791]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:05.077 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:05.893 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:05 smithi040 ceph-mon[97954]: pgmap v319: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:06.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:05 smithi130 ceph-mon[129405]: pgmap v319: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:06.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:05 smithi045 ceph-mon[128791]: pgmap v319: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 10 op/s 2023-07-14T11:11:06.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.517+0000 7f1b096ac700 1 -- 172.21.15.40:0/1221708985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 msgr2=0x7f1b0410e7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.517+0000 7f1b096ac700 1 --2- 172.21.15.40:0/1221708985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0410e7c0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f1afc00b0a0 tx=0x7f1afc00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 -- 172.21.15.40:0/1221708985 shutdown_connections 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 --2- 172.21.15.40:0/1221708985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0410e7c0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 --2- 172.21.15.40:0/1221708985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040728b0 0x7f1b04072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 --2- 172.21.15.40:0/1221708985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040731c0 0x7f1b0410bea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 -- 172.21.15.40:0/1221708985 >> 172.21.15.40:0/1221708985 conn(0x7f1b040fb660 msgr2=0x7f1b040fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:06.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 -- 172.21.15.40:0/1221708985 shutdown_connections 2023-07-14T11:11:06.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.518+0000 7f1b096ac700 1 -- 172.21.15.40:0/1221708985 wait complete. 2023-07-14T11:11:06.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 Processor -- start 2023-07-14T11:11:06.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 -- start start 2023-07-14T11:11:06.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:06.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040731c0 0x7f1b04198450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:06.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0419c8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:06.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b0419cf60 con 0x7f1b0410c3e0 2023-07-14T11:11:06.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b0419d0d0 con 0x7f1b040728b0 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.519+0000 7f1b096ac700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1b0419d290 con 0x7f1b040731c0 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b02ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b02ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35470/0 (socket says 172.21.15.40:35470) 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b02ffd700 1 -- 172.21.15.40:0/3663306647 learned_addr learned my addr 172.21.15.40:0/3663306647 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0419c8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b027fc700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040731c0 0x7f1b04198450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:06.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 -- 172.21.15.40:0/3663306647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040731c0 msgr2=0x7f1b04198450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040731c0 0x7f1b04198450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 -- 172.21.15.40:0/3663306647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 msgr2=0x7f1b04197f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 -- 172.21.15.40:0/3663306647 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1afc009d00 con 0x7f1b0410c3e0 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b02ffd700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.520+0000 7f1b037fe700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0419c8c0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f1afc009fd0 tx=0x7f1afc00bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.521+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1afc00e040 con 0x7f1b0410c3e0 2023-07-14T11:11:06.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.521+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b0419d510 con 0x7f1b0410c3e0 2023-07-14T11:11:06.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.521+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1afc0094c0 con 0x7f1b0410c3e0 2023-07-14T11:11:06.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.521+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1afc005040 con 0x7f1b0410c3e0 2023-07-14T11:11:06.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.521+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1b041a4d40 con 0x7f1b0410c3e0 2023-07-14T11:11:06.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.522+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1b04106d00 con 0x7f1b0410c3e0 2023-07-14T11:11:06.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.524+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1afc01d470 con 0x7f1b0410c3e0 2023-07-14T11:11:06.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.526+0000 7f1aebfff700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 0x7f1aec0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:06.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.526+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7f1afc0ce160 con 0x7f1b0410c3e0 2023-07-14T11:11:06.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.527+0000 7f1b02ffd700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 0x7f1aec0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:06.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.527+0000 7f1b02ffd700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 0x7f1aec0a3600 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f1af4011550 tx=0x7f1af4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:06.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.530+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1afc0975d0 con 0x7f1b0410c3e0 2023-07-14T11:11:06.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.787+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f1b0404e210 con 0x7f1b0410c3e0 2023-07-14T11:11:06.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.788+0000 7f1aebfff700 1 -- 172.21.15.40:0/3663306647 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f1afc09abf0 con 0x7f1b0410c3e0 2023-07-14T11:11:06.791 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:06.791 INFO:teuthology.orchestra.run.smithi040.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-07-14T11:11:06.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 msgr2=0x7f1aec0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:06.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 0x7f1aec0a3600 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7f1af4011550 tx=0x7f1af4009260 comp rx=0 tx=0).stop 2023-07-14T11:11:06.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 msgr2=0x7f1b0419c8c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:06.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0419c8c0 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f1afc009fd0 tx=0x7f1afc00bce0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 shutdown_connections 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1aec0a1150 0x7f1aec0a3600 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1b0410c3e0 0x7f1b0419c8c0 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1b040728b0 0x7f1b04197f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 --2- 172.21.15.40:0/3663306647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1b040731c0 0x7f1b04198450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.790+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 >> 172.21.15.40:0/3663306647 conn(0x7f1b040fb660 msgr2=0x7f1b04103e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:06.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.791+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 shutdown_connections 2023-07-14T11:11:06.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:06.791+0000 7f1b096ac700 1 -- 172.21.15.40:0/3663306647 wait complete. 2023-07-14T11:11:07.276 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-07-14T11:11:07.277 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:07.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 -- 172.21.15.40:0/2403864246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 msgr2=0x7fcba8107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 --2- 172.21.15.40:0/2403864246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba8107f20 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fcba000b0a0 tx=0x7fcba000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 -- 172.21.15.40:0/2403864246 shutdown_connections 2023-07-14T11:11:07.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 --2- 172.21.15.40:0/2403864246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba8107f20 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 --2- 172.21.15.40:0/2403864246 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba80ffae0 0x7fcba80ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 --2- 172.21.15.40:0/2403864246 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba8100cd0 0x7fcba8101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 -- 172.21.15.40:0/2403864246 >> 172.21.15.40:0/2403864246 conn(0x7fcba80fb100 msgr2=0x7fcba80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:07.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 -- 172.21.15.40:0/2403864246 shutdown_connections 2023-07-14T11:11:07.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.483+0000 7fcbaffff700 1 -- 172.21.15.40:0/2403864246 wait complete. 2023-07-14T11:11:07.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.484+0000 7fcbaffff700 1 Processor -- start 2023-07-14T11:11:07.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.484+0000 7fcbaffff700 1 -- start start 2023-07-14T11:11:07.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 0x7fcba8197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba8100cd0 0x7fcba8197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcba819f600 con 0x7fcba8101660 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcba819f740 con 0x7fcba80ffae0 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbaffff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcba819f8d0 con 0x7fcba8100cd0 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbadd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 0x7fcba8197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37254/0 (socket says 172.21.15.40:37254) 2023-07-14T11:11:07.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbadd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 0x7fcba8197a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35494/0 (socket says 172.21.15.40:35494) 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 -- 172.21.15.40:0/3341264216 learned_addr learned my addr 172.21.15.40:0/3341264216 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbad59a700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba8100cd0 0x7fcba8197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 -- 172.21.15.40:0/3341264216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba8100cd0 msgr2=0x7fcba8197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba8100cd0 0x7fcba8197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 -- 172.21.15.40:0/3341264216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 msgr2=0x7fcba8197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 0x7fcba8197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.485+0000 7fcbae59c700 1 -- 172.21.15.40:0/3341264216 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcba0009d00 con 0x7fcba8101660 2023-07-14T11:11:07.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcbae59c700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fcba000d010 tx=0x7fcba000bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:07.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcba000e040 con 0x7fcba8101660 2023-07-14T11:11:07.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcba0003e10 con 0x7fcba8101660 2023-07-14T11:11:07.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcba819fb50 con 0x7fcba8101660 2023-07-14T11:11:07.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcba0005260 con 0x7fcba8101660 2023-07-14T11:11:07.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.486+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcba819ff20 con 0x7fcba8101660 2023-07-14T11:11:07.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.487+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcba810b380 con 0x7fcba8101660 2023-07-14T11:11:07.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.489+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fcba00053e0 con 0x7fcba8101660 2023-07-14T11:11:07.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.491+0000 7fcb9effd700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 0x7fcb940a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.492+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(90..90 src has 1..90) v4 ==== 9020+0+0 (secure 0 0 0) 0x7fcba00ce110 con 0x7fcba8101660 2023-07-14T11:11:07.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.492+0000 7fcbadd9b700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 0x7fcb940a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.492+0000 7fcbadd9b700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 0x7fcb940a3650 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fcba8100b30 tx=0x7fcba4007400 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:07.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.496+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcba0097580 con 0x7fcba8101660 2023-07-14T11:11:07.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.715+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcba81a04d0 con 0x7fcba8101660 2023-07-14T11:11:07.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.717+0000 7fcb9effd700 1 -- 172.21.15.40:0/3341264216 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v91) v1 ==== 74+0+24481 (secure 0 0 0) 0x7fcba009aba0 con 0x7fcba8101660 2023-07-14T11:11:07.720 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:07.720 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":91,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:07.704300+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":82,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:07.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 msgr2=0x7fcb940a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 0x7fcb940a3650 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fcba8100b30 tx=0x7fcba4007400 comp rx=0 tx=0).stop 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 msgr2=0x7fcba819f030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fcba000d010 tx=0x7fcba000bc80 comp rx=0 tx=0).stop 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 shutdown_connections 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcb940a11a0 0x7fcb940a3650 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcba8101660 0x7fcba819f030 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcba80ffae0 0x7fcba8197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 --2- 172.21.15.40:0/3341264216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcba8100cd0 0x7fcba8197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 >> 172.21.15.40:0/3341264216 conn(0x7fcba80fb100 msgr2=0x7fcba8106510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:07.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 shutdown_connections 2023-07-14T11:11:07.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.720+0000 7fcbaffff700 1 -- 172.21.15.40:0/3341264216 wait complete. 2023-07-14T11:11:07.742 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:07.742 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:07.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 -- 172.21.15.40:0/3949560222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660101660 msgr2=0x7ff660107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 --2- 172.21.15.40:0/3949560222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660101660 0x7ff660107f20 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7ff65800b0a0 tx=0x7ff65800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 -- 172.21.15.40:0/3949560222 shutdown_connections 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 --2- 172.21.15.40:0/3949560222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660101660 0x7ff660107f20 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 --2- 172.21.15.40:0/3949560222 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff6600ffae0 0x7ff6600ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 --2- 172.21.15.40:0/3949560222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff660100cd0 0x7ff660101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 -- 172.21.15.40:0/3949560222 >> 172.21.15.40:0/3949560222 conn(0x7ff6600fb0e0 msgr2=0x7ff6600fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 -- 172.21.15.40:0/3949560222 shutdown_connections 2023-07-14T11:11:07.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.946+0000 7ff667bff700 1 -- 172.21.15.40:0/3949560222 wait complete. 2023-07-14T11:11:07.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 Processor -- start 2023-07-14T11:11:07.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 -- start start 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 0x7ff660195730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 0x7ff66019cce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff66019d2b0 con 0x7ff6600ffae0 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff66019d3f0 con 0x7ff660101660 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.948+0000 7ff667bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff66019d580 con 0x7ff660100cd0 2023-07-14T11:11:07.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 0x7ff660195730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66519a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66519a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37054/0 (socket says 172.21.15.40:37054) 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66519a700 1 -- 172.21.15.40:0/2111646023 learned_addr learned my addr 172.21.15.40:0/2111646023 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66619c700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 0x7ff66019cce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 -- 172.21.15.40:0/2111646023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 msgr2=0x7ff660195c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 -- 172.21.15.40:0/2111646023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 msgr2=0x7ff66019cce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:07.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 0x7ff66019cce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 -- 172.21.15.40:0/2111646023 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff65c009520 con 0x7ff6600ffae0 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66519a700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66619c700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 0x7ff66019cce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.949+0000 7ff66599b700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 0x7ff660195730 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff65c00e9c0 tx=0x7ff65c00ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.950+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff65c00cb70 con 0x7ff6600ffae0 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.950+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff650009520 con 0x7ff6600ffae0 2023-07-14T11:11:07.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.950+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff65c00ee60 con 0x7ff6600ffae0 2023-07-14T11:11:07.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.950+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff65c018680 con 0x7ff6600ffae0 2023-07-14T11:11:07.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.950+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff658009d00 con 0x7ff6600ffae0 2023-07-14T11:11:07.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.951+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff660106990 con 0x7ff6600ffae0 2023-07-14T11:11:07.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.953+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7ff65c010430 con 0x7ff6600ffae0 2023-07-14T11:11:07.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.955+0000 7ff656ffd700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 0x7ff64c0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:07.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.955+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(91..91 src has 1..91) v4 ==== 9086+0+0 (secure 0 0 0) 0x7ff65c014070 con 0x7ff6600ffae0 2023-07-14T11:11:07.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.955+0000 7ff66519a700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 0x7ff64c0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:07.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.956+0000 7ff66519a700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 0x7ff64c0a3600 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7ff650009fd0 tx=0x7ff650009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:07.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:07.959+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff65c096410 con 0x7ff6600ffae0 2023-07-14T11:11:08.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:07 smithi130 ceph-mon[129405]: pgmap v320: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]: dispatch 2023-07-14T11:11:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:07 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]: dispatch 2023-07-14T11:11:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:07 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3663306647' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-07-14T11:11:08.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:07 smithi040 ceph-mon[97954]: pgmap v320: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:08.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]: dispatch 2023-07-14T11:11:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:07 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]: dispatch 2023-07-14T11:11:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:07 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3663306647' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-07-14T11:11:08.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:07 smithi045 ceph-mon[128791]: pgmap v320: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:08.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]: dispatch 2023-07-14T11:11:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:07 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]: dispatch 2023-07-14T11:11:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:07 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3663306647' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-07-14T11:11:08.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.169+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7ff660061120 con 0x7ff6600ffae0 2023-07-14T11:11:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.169+0000 7ff656ffd700 1 -- 172.21.15.40:0/2111646023 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v91) v1 ==== 93+0+10 (secure 0 0 0) 0x7ff65c099a30 con 0x7ff6600ffae0 2023-07-14T11:11:08.172 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 msgr2=0x7ff64c0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 0x7ff64c0a3600 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7ff650009fd0 tx=0x7ff650009260 comp rx=0 tx=0).stop 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 msgr2=0x7ff660195730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 0x7ff660195730 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7ff65c00e9c0 tx=0x7ff65c00ecd0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 shutdown_connections 2023-07-14T11:11:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff64c0a1150 0x7ff64c0a3600 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff660101660 0x7ff66019cce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6600ffae0 0x7ff660195730 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 --2- 172.21.15.40:0/2111646023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff660100cd0 0x7ff660195c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 >> 172.21.15.40:0/2111646023 conn(0x7ff6600fb0e0 msgr2=0x7ff6600fd300 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 shutdown_connections 2023-07-14T11:11:08.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.172+0000 7ff667bff700 1 -- 172.21.15.40:0/2111646023 wait complete. 2023-07-14T11:11:08.193 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:08.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 -- 172.21.15.40:0/3777008835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754100cd0 msgr2=0x7f4754101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 --2- 172.21.15.40:0/3777008835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754100cd0 0x7f4754101120 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f473c009990 tx=0x7f473c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 -- 172.21.15.40:0/3777008835 shutdown_connections 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 --2- 172.21.15.40:0/3777008835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4754101660 0x7f4754107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 --2- 172.21.15.40:0/3777008835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f47540ffae0 0x7f47540ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 --2- 172.21.15.40:0/3777008835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754100cd0 0x7f4754101120 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.397+0000 7f4758dff700 1 -- 172.21.15.40:0/3777008835 >> 172.21.15.40:0/3777008835 conn(0x7f47540fb0e0 msgr2=0x7f47540fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:08.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.398+0000 7f4758dff700 1 -- 172.21.15.40:0/3777008835 shutdown_connections 2023-07-14T11:11:08.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.398+0000 7f4758dff700 1 -- 172.21.15.40:0/3777008835 wait complete. 2023-07-14T11:11:08.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 Processor -- start 2023-07-14T11:11:08.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 -- start start 2023-07-14T11:11:08.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47540ffae0 0x7f4754197a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475419f610 con 0x7f4754101660 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475419f750 con 0x7f47540ffae0 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.399+0000 7f4758dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475419f8e0 con 0x7f4754100cd0 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f4752d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f475259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47540ffae0 0x7f4754197a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f4752d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37278/0 (socket says 172.21.15.40:37278) 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37084/0 (socket says 172.21.15.40:37084) 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f4752d9d700 1 -- 172.21.15.40:0/3514138436 learned_addr learned my addr 172.21.15.40:0/3514138436 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 -- 172.21.15.40:0/3514138436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47540ffae0 msgr2=0x7f4754197a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47540ffae0 0x7f4754197a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 -- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 msgr2=0x7f475419f040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 -- 172.21.15.40:0/3514138436 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f473c0095f0 con 0x7f4754100cd0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f4752d9d700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474bfff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f473c0098e0 tx=0x7f473c005740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.400+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f473c01d070 con 0x7f4754100cd0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.401+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f475419fb60 con 0x7f4754100cd0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.401+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f473c021410 con 0x7f4754100cd0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.401+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f473c017650 con 0x7f4754100cd0 2023-07-14T11:11:08.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.401+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f475419ffe0 con 0x7f4754100cd0 2023-07-14T11:11:08.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.402+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f475410b300 con 0x7f4754100cd0 2023-07-14T11:11:08.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.404+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f473c021580 con 0x7f4754100cd0 2023-07-14T11:11:08.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.406+0000 7f474b7fe700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 0x7f47400a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.406+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(91..91 src has 1..91) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f473c0ce650 con 0x7f4754100cd0 2023-07-14T11:11:08.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.406+0000 7f475259c700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 0x7f47400a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.406+0000 7f475259c700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 0x7f47400a3600 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f4754100b30 tx=0x7f4744009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:08.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.407+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f473c097a80 con 0x7f4754100cd0 2023-07-14T11:11:08.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.617+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f47541a05f0 con 0x7f4754100cd0 2023-07-14T11:11:08.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.618+0000 7f474b7fe700 1 -- 172.21.15.40:0/3514138436 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v91) v1 ==== 104+0+11 (secure 0 0 0) 0x7f473c09b0a0 con 0x7f4754100cd0 2023-07-14T11:11:08.620 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:08.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 msgr2=0x7f47400a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 0x7f47400a3600 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f4754100b30 tx=0x7f4744009260 comp rx=0 tx=0).stop 2023-07-14T11:11:08.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 msgr2=0x7f4754197fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f473c0098e0 tx=0x7f473c005740 comp rx=0 tx=0).stop 2023-07-14T11:11:08.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 shutdown_connections 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47400a1150 0x7f47400a3600 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4754101660 0x7f475419f040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f47540ffae0 0x7f4754197a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 --2- 172.21.15.40:0/3514138436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4754100cd0 0x7f4754197fd0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 >> 172.21.15.40:0/3514138436 conn(0x7f47540fb0e0 msgr2=0x7f47540fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 shutdown_connections 2023-07-14T11:11:08.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.620+0000 7f4758dff700 1 -- 172.21.15.40:0/3514138436 wait complete. 2023-07-14T11:11:08.641 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:08.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.848+0000 7f59af1ff700 1 -- 172.21.15.40:0/1168984268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a81079b0 msgr2=0x7f59a8109e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.848+0000 7f59af1ff700 1 --2- 172.21.15.40:0/1168984268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a81079b0 0x7f59a8109e30 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f59a000b0a0 tx=0x7f59a000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 -- 172.21.15.40:0/1168984268 shutdown_connections 2023-07-14T11:11:08.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 --2- 172.21.15.40:0/1168984268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a81079b0 0x7f59a8109e30 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 --2- 172.21.15.40:0/1168984268 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a8074550 0x7f59a80729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 --2- 172.21.15.40:0/1168984268 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a80733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 -- 172.21.15.40:0/1168984268 >> 172.21.15.40:0/1168984268 conn(0x7f59a80fb130 msgr2=0x7f59a80fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:08.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 -- 172.21.15.40:0/1168984268 shutdown_connections 2023-07-14T11:11:08.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.849+0000 7f59af1ff700 1 -- 172.21.15.40:0/1168984268 wait complete. 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 Processor -- start 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 -- start start 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 0x7f59a81980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a81079b0 0x7f59a819f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59a819f6e0 con 0x7f59a8074550 2023-07-14T11:11:08.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59a819f820 con 0x7f59a8072f70 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.850+0000 7f59af1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59a819f9b0 con 0x7f59a81079b0 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 0x7f59a81980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59acf9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59acf9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35558/0 (socket says 172.21.15.40:35558) 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59acf9b700 1 -- 172.21.15.40:0/2568925394 learned_addr learned my addr 172.21.15.40:0/2568925394 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59ad79c700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a81079b0 0x7f59a819f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 -- 172.21.15.40:0/2568925394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a81079b0 msgr2=0x7f59a819f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a81079b0 0x7f59a819f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 -- 172.21.15.40:0/2568925394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 msgr2=0x7f59a8197b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 -- 172.21.15.40:0/2568925394 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5994009520 con 0x7f59a8074550 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59acf9b700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.851+0000 7f59a7fff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 0x7f59a81980a0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f599400e9c0 tx=0x7f599400ecd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.852+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f599400cb70 con 0x7f59a8074550 2023-07-14T11:11:08.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.852+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f59a0009d00 con 0x7f59a8074550 2023-07-14T11:11:08.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.852+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f599400ee60 con 0x7f59a8074550 2023-07-14T11:11:08.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.852+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5994018680 con 0x7f59a8074550 2023-07-14T11:11:08.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.852+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f59a81a0060 con 0x7f59a8074550 2023-07-14T11:11:08.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.853+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f59a8106060 con 0x7f59a8074550 2023-07-14T11:11:08.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.855+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5994010430 con 0x7f59a8074550 2023-07-14T11:11:08.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.856+0000 7f59a5ffb700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 0x7f59980a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:08.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.856+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5994014070 con 0x7f59a8074550 2023-07-14T11:11:08.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.856+0000 7f59acf9b700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 0x7f59980a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:08.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.857+0000 7f59acf9b700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 0x7f59980a3680 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f599c005950 tx=0x7f599c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:08.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:08.860+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5994096350 con 0x7f59a8074550 2023-07-14T11:11:09.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]': finished 2023-07-14T11:11:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]': finished 2023-07-14T11:11:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: osdmap e91: 12 total, 12 up, 12 in 2023-07-14T11:11:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3341264216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2111646023' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3514138436' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]': finished 2023-07-14T11:11:09.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]': finished 2023-07-14T11:11:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: osdmap e91: 12 total, 12 up, 12 in 2023-07-14T11:11:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3341264216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2111646023' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3514138436' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [9, 1]}]': finished 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: from='mgr.14148 172.21.15.40:0/2405753418' entity='mgr.x' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [9, 10, 5, 2]}]': finished 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: osdmap e91: 12 total, 12 up, 12 in 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3341264216' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2111646023' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3514138436' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.066+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f59a802cce0 con 0x7f59a8074550 2023-07-14T11:11:09.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.066+0000 7f59a5ffb700 1 -- 172.21.15.40:0/2568925394 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7f599400ce60 con 0x7f59a8074550 2023-07-14T11:11:09.069 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:09.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.068+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 msgr2=0x7f59980a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.068+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 0x7f59980a3680 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f599c005950 tx=0x7f599c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:09.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.068+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 msgr2=0x7f59a81980a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.068+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 0x7f59a81980a0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f599400e9c0 tx=0x7f599400ecd0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 shutdown_connections 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f59980a11d0 0x7f59980a3680 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f59a81079b0 0x7f59a819f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f59a8072f70 0x7f59a8197b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 --2- 172.21.15.40:0/2568925394 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f59a8074550 0x7f59a81980a0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 >> 172.21.15.40:0/2568925394 conn(0x7f59a80fb130 msgr2=0x7f59a8109bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:09.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 shutdown_connections 2023-07-14T11:11:09.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.069+0000 7f59af1ff700 1 -- 172.21.15.40:0/2568925394 wait complete. 2023-07-14T11:11:09.090 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:09.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/2822282426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f80ffae0 msgr2=0x7fa8f80ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/2822282426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f80ffae0 0x7fa8f80ffef0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fa8f4009990 tx=0x7fa8f4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/2822282426 shutdown_connections 2023-07-14T11:11:09.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/2822282426 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8101660 0x7fa8f8107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/2822282426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f80ffae0 0x7fa8f80ffef0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/2822282426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/2822282426 >> 172.21.15.40:0/2822282426 conn(0x7fa8f80fb0e0 msgr2=0x7fa8f80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:09.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/2822282426 shutdown_connections 2023-07-14T11:11:09.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.297+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/2822282426 wait complete. 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.298+0000 7fa8ff5ff700 1 Processor -- start 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.298+0000 7fa8ff5ff700 1 -- start start 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f80ffae0 0x7fa8f8197a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8f819f610 con 0x7fa8f8101660 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8f819f750 con 0x7fa8f8100cd0 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8ff5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8f819f8e0 con 0x7fa8f80ffae0 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fcb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37330/0 (socket says 172.21.15.40:37330) 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fcb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35572/0 (socket says 172.21.15.40:35572) 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 -- 172.21.15.40:0/669798819 learned_addr learned my addr 172.21.15.40:0/669798819 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:09.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fd39b700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f80ffae0 0x7fa8f8197a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 -- 172.21.15.40:0/669798819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f80ffae0 msgr2=0x7fa8f8197a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f80ffae0 0x7fa8f8197a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 -- 172.21.15.40:0/669798819 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 msgr2=0x7fa8f8197fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fdb9c700 1 -- 172.21.15.40:0/669798819 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa8f40095f0 con 0x7fa8f8101660 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.299+0000 7fa8fcb9a700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:09.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8fdb9c700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa8f0007ee0 tx=0x7fa8f000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8f000ddd0 con 0x7fa8f8101660 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa8f000f040 con 0x7fa8f8101660 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8f819fbc0 con 0x7fa8f8101660 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa8f00048e0 con 0x7fa8f8101660 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.300+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8f81a0120 con 0x7fa8f8101660 2023-07-14T11:11:09.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.301+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa8f810b300 con 0x7fa8f8101660 2023-07-14T11:11:09.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.303+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa8f0003a00 con 0x7fa8f8101660 2023-07-14T11:11:09.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.305+0000 7fa8ee7fc700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 0x7fa8e40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.305+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fa8f00cc540 con 0x7fa8f8101660 2023-07-14T11:11:09.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.305+0000 7fa8fd39b700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 0x7fa8e40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.306+0000 7fa8fd39b700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 0x7fa8e40a3680 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fa8f400b2e0 tx=0x7fa8f4005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:09.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.308+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fa8f004e790 con 0x7fa8f8101660 2023-07-14T11:11:09.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.520+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7fa8f81a06d0 con 0x7fa8f8101660 2023-07-14T11:11:09.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.520+0000 7fa8ee7fc700 1 -- 172.21.15.40:0/669798819 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7fa8f00958e0 con 0x7fa8f8101660 2023-07-14T11:11:09.523 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 msgr2=0x7fa8e40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 0x7fa8e40a3680 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7fa8f400b2e0 tx=0x7fa8f4005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 msgr2=0x7fa8f819f040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fa8f0007ee0 tx=0x7fa8f000d3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 shutdown_connections 2023-07-14T11:11:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa8e40a11d0 0x7fa8e40a3680 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8f8101660 0x7fa8f819f040 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa8f80ffae0 0x7fa8f8197a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 --2- 172.21.15.40:0/669798819 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa8f8100cd0 0x7fa8f8197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 >> 172.21.15.40:0/669798819 conn(0x7fa8f80fb0e0 msgr2=0x7fa8f80fd3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 shutdown_connections 2023-07-14T11:11:09.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.523+0000 7fa8ff5ff700 1 -- 172.21.15.40:0/669798819 wait complete. 2023-07-14T11:11:09.544 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:09.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 -- 172.21.15.40:0/1831720470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 msgr2=0x7f4f080fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 --2- 172.21.15.40:0/1831720470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080fff10 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f4f04009990 tx=0x7f4f04009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 -- 172.21.15.40:0/1831720470 shutdown_connections 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 --2- 172.21.15.40:0/1831720470 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 --2- 172.21.15.40:0/1831720470 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080fff10 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 --2- 172.21.15.40:0/1831720470 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 0x7f4f08101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.746+0000 7f4f103ff700 1 -- 172.21.15.40:0/1831720470 >> 172.21.15.40:0/1831720470 conn(0x7f4f080fb0e0 msgr2=0x7f4f080fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.747+0000 7f4f103ff700 1 -- 172.21.15.40:0/1831720470 shutdown_connections 2023-07-14T11:11:09.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.747+0000 7f4f103ff700 1 -- 172.21.15.40:0/1831720470 wait complete. 2023-07-14T11:11:09.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.748+0000 7f4f103ff700 1 Processor -- start 2023-07-14T11:11:09.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.748+0000 7f4f103ff700 1 -- start start 2023-07-14T11:11:09.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 0x7f4f08072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f080735e0 con 0x7f4f08101680 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f08073750 con 0x7f4f08100cf0 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f103ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f0819f810 con 0x7f4f080ffb00 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0d99a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 0x7f4f08072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37344/0 (socket says 172.21.15.40:37344) 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 -- 172.21.15.40:0/846918267 learned_addr learned my addr 172.21.15.40:0/846918267 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e19b700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 -- 172.21.15.40:0/846918267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 msgr2=0x7f4f080742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 -- 172.21.15.40:0/846918267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 msgr2=0x7f4f08072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:09.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 0x7f4f08072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.749+0000 7f4f0e99c700 1 -- 172.21.15.40:0/846918267 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f00009c40 con 0x7f4f08101680 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4f0e99c700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f4f0000c370 tx=0x7f4f0000c680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f00017070 con 0x7f4f08101680 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4f0000f040 con 0x7f4f08101680 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f040095f0 con 0x7f4f08101680 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4f0000e050 con 0x7f4f08101680 2023-07-14T11:11:09.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.750+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f0819fe60 con 0x7f4f08101680 2023-07-14T11:11:09.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.751+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f081070a0 con 0x7f4f08101680 2023-07-14T11:11:09.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.753+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4f00004520 con 0x7f4f08101680 2023-07-14T11:11:09.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.755+0000 7f4eff7fe700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 0x7f4ef40a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:09.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.755+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4f000ccc30 con 0x7f4f08101680 2023-07-14T11:11:09.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.755+0000 7f4f0e19b700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 0x7f4ef40a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:09.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.756+0000 7f4f0e19b700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 0x7f4ef40a36d0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f4f04006010 tx=0x7f4f0400b260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:09.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:09.757+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4f00095fd0 con 0x7f4f08101680 2023-07-14T11:11:09.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:09 smithi040 ceph-mon[97954]: pgmap v322: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:09 smithi040 ceph-mon[97954]: osdmap e92: 12 total, 12 up, 12 in 2023-07-14T11:11:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2568925394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/669798819' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:10.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.011+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f4f0802c850 con 0x7f4f08101680 2023-07-14T11:11:10.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.012+0000 7f4eff7fe700 1 -- 172.21.15.40:0/846918267 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f4f000995f0 con 0x7f4f08101680 2023-07-14T11:11:10.015 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:10.015 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:10.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.014+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 msgr2=0x7f4ef40a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.014+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 0x7f4ef40a36d0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f4f04006010 tx=0x7f4f0400b260 comp rx=0 tx=0).stop 2023-07-14T11:11:10.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.014+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 msgr2=0x7f4f08072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.014+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f4f0000c370 tx=0x7f4f0000c680 comp rx=0 tx=0).stop 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 shutdown_connections 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ef40a1220 0x7f4ef40a36d0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f08101680 0x7f4f08072e40 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4f080ffb00 0x7f4f080742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 --2- 172.21.15.40:0/846918267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4f08100cf0 0x7f4f08072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 >> 172.21.15.40:0/846918267 conn(0x7f4f080fb0e0 msgr2=0x7f4f081041e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:10.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 shutdown_connections 2023-07-14T11:11:10.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.015+0000 7f4f103ff700 1 -- 172.21.15.40:0/846918267 wait complete. 2023-07-14T11:11:10.019 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:10.037 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:09 smithi130 ceph-mon[129405]: pgmap v322: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:09 smithi130 ceph-mon[129405]: osdmap e92: 12 total, 12 up, 12 in 2023-07-14T11:11:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2568925394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/669798819' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:09 smithi045 ceph-mon[128791]: pgmap v322: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:09 smithi045 ceph-mon[128791]: osdmap e92: 12 total, 12 up, 12 in 2023-07-14T11:11:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2568925394' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/669798819' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:10.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.236+0000 7fc34a9ff700 1 -- 172.21.15.40:0/912623536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc344101680 msgr2=0x7fc344107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.236+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/912623536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc344101680 0x7fc344107f40 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7fc33c00b0a0 tx=0x7fc33c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 -- 172.21.15.40:0/912623536 shutdown_connections 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/912623536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc344101680 0x7fc344107f40 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/912623536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3440ffb00 0x7fc3440fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/912623536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344100cf0 0x7fc344101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 -- 172.21.15.40:0/912623536 >> 172.21.15.40:0/912623536 conn(0x7fc3440fb0e0 msgr2=0x7fc3440fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 -- 172.21.15.40:0/912623536 shutdown_connections 2023-07-14T11:11:10.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.237+0000 7fc34a9ff700 1 -- 172.21.15.40:0/912623536 wait complete. 2023-07-14T11:11:10.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 Processor -- start 2023-07-14T11:11:10.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 -- start start 2023-07-14T11:11:10.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc344100cf0 0x7fc344198050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344101680 0x7fc34419f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc34419f690 con 0x7fc344101680 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc34419f7d0 con 0x7fc3440ffb00 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.238+0000 7fc34a9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc34419f960 con 0x7fc344100cf0 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35612/0 (socket says 172.21.15.40:35612) 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 -- 172.21.15.40:0/871872536 learned_addr learned my addr 172.21.15.40:0/871872536 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:10.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc3437fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc344100cf0 0x7fc344198050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc348f9c700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344101680 0x7fc34419f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 -- 172.21.15.40:0/871872536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc344100cf0 msgr2=0x7fc344198050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc344100cf0 0x7fc344198050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 -- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344101680 msgr2=0x7fc34419f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344101680 0x7fc34419f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 -- 172.21.15.40:0/871872536 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc33c009d00 con 0x7fc3440ffb00 2023-07-14T11:11:10.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc343fff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fc33400e970 tx=0x7fc33400ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:10.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc334009790 con 0x7fc3440ffb00 2023-07-14T11:11:10.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc33400c7e0 con 0x7fc3440ffb00 2023-07-14T11:11:10.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc334003c70 con 0x7fc3440ffb00 2023-07-14T11:11:10.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.239+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc34419fc40 con 0x7fc3440ffb00 2023-07-14T11:11:10.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.240+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3441a0100 con 0x7fc3440ffb00 2023-07-14T11:11:10.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.241+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc34410b330 con 0x7fc3440ffb00 2023-07-14T11:11:10.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.243+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc33400cc50 con 0x7fc3440ffb00 2023-07-14T11:11:10.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.244+0000 7fc3417fa700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 0x7fc32c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.244+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc334014070 con 0x7fc3440ffb00 2023-07-14T11:11:10.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.244+0000 7fc3437fe700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 0x7fc32c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.245+0000 7fc3437fe700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 0x7fc32c0a3650 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fc338009c60 tx=0x7fc338009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:10.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.246+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc334096800 con 0x7fc3440ffb00 2023-07-14T11:11:10.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.456+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fc34410b550 con 0x7fc3440ffb00 2023-07-14T11:11:10.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.459+0000 7fc3417fa700 1 -- 172.21.15.40:0/871872536 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7fc334003dd0 con 0x7fc3440ffb00 2023-07-14T11:11:10.462 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:10.462 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:10.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 msgr2=0x7fc32c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 0x7fc32c0a3650 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7fc338009c60 tx=0x7fc338009260 comp rx=0 tx=0).stop 2023-07-14T11:11:10.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 msgr2=0x7fc344197b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fc33400e970 tx=0x7fc33400ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 shutdown_connections 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc32c0a11a0 0x7fc32c0a3650 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc344101680 0x7fc34419f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3440ffb00 0x7fc344197b10 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 --2- 172.21.15.40:0/871872536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc344100cf0 0x7fc344198050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 >> 172.21.15.40:0/871872536 conn(0x7fc3440fb0e0 msgr2=0x7fc3440fd220 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 shutdown_connections 2023-07-14T11:11:10.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.461+0000 7fc34a9ff700 1 -- 172.21.15.40:0/871872536 wait complete. 2023-07-14T11:11:10.483 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:10.484 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 -- 172.21.15.40:0/3840431959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4858100cd0 msgr2=0x7f4858101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 --2- 172.21.15.40:0/3840431959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4858100cd0 0x7f4858101120 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f484c009990 tx=0x7f484c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 -- 172.21.15.40:0/3840431959 shutdown_connections 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 --2- 172.21.15.40:0/3840431959 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858101660 0x7f4858107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 --2- 172.21.15.40:0/3840431959 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48580ffae0 0x7f48580ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 --2- 172.21.15.40:0/3840431959 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4858100cd0 0x7f4858101120 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 -- 172.21.15.40:0/3840431959 >> 172.21.15.40:0/3840431959 conn(0x7f48580fb0e0 msgr2=0x7f48580fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:10.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 -- 172.21.15.40:0/3840431959 shutdown_connections 2023-07-14T11:11:10.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.690+0000 7f485e7ff700 1 -- 172.21.15.40:0/3840431959 wait complete. 2023-07-14T11:11:10.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.691+0000 7f485e7ff700 1 Processor -- start 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.691+0000 7f485e7ff700 1 -- start start 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.691+0000 7f485e7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48580ffae0 0x7f48581979e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485e7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858100cd0 0x7f4858197f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485e7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485e7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f485819f560 con 0x7f4858101660 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485e7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f485819f6a0 con 0x7f4858100cd0 2023-07-14T11:11:10.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485e7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f485819f830 con 0x7f48580ffae0 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37372/0 (socket says 172.21.15.40:37372) 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 -- 172.21.15.40:0/1433517109 learned_addr learned my addr 172.21.15.40:0/1433517109 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f48577fe700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858100cd0 0x7f4858197f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f4857fff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48580ffae0 0x7f48581979e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 -- 172.21.15.40:0/1433517109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48580ffae0 msgr2=0x7f48581979e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48580ffae0 0x7f48581979e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 -- 172.21.15.40:0/1433517109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858100cd0 msgr2=0x7f4858197f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858100cd0 0x7f4858197f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.692+0000 7f485cd9c700 1 -- 172.21.15.40:0/1433517109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f484c0095f0 con 0x7f4858101660 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f485cd9c700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f485000ef30 tx=0x7f485000c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4850018070 con 0x7f4858101660 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f485819fab0 con 0x7f4858101660 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f485000f040 con 0x7f4858101660 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f485000cd00 con 0x7f4858101660 2023-07-14T11:11:10.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.693+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f485819ffc0 con 0x7f4858101660 2023-07-14T11:11:10.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.694+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f485810b250 con 0x7f4858101660 2023-07-14T11:11:10.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.696+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4850003a00 con 0x7f4858101660 2023-07-14T11:11:10.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.698+0000 7f48557fa700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 0x7f48400a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:10.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.698+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f48500cc810 con 0x7f4858101660 2023-07-14T11:11:10.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.698+0000 7f4857fff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 0x7f48400a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:10.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.699+0000 7f4857fff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 0x7f48400a3600 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f4858100b30 tx=0x7f4848009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:10.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.702+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4850095c40 con 0x7f4858101660 2023-07-14T11:11:10.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:10 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/846918267' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:10.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:10 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/871872536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:10.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.916+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f48581a05d0 con 0x7f4858101660 2023-07-14T11:11:10.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.916+0000 7f48557fa700 1 -- 172.21.15.40:0/1433517109 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7f4850099260 con 0x7f4858101660 2023-07-14T11:11:10.919 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 msgr2=0x7f48400a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 0x7f48400a3600 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f4858100b30 tx=0x7f4848009260 comp rx=0 tx=0).stop 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 msgr2=0x7f485819ef90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f485000ef30 tx=0x7f485000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 shutdown_connections 2023-07-14T11:11:10.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f48400a1150 0x7f48400a3600 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4858101660 0x7f485819ef90 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48580ffae0 0x7f48581979e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 --2- 172.21.15.40:0/1433517109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4858100cd0 0x7f4858197f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 >> 172.21.15.40:0/1433517109 conn(0x7f48580fb0e0 msgr2=0x7f48580fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 shutdown_connections 2023-07-14T11:11:10.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:10.919+0000 7f485e7ff700 1 -- 172.21.15.40:0/1433517109 wait complete. 2023-07-14T11:11:10.941 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:11.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:10 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/846918267' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:11.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:10 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/871872536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:11.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:10 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/846918267' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:10 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/871872536' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:11.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 -- 172.21.15.40:0/3888044841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c00ffae0 msgr2=0x7f85c00ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3888044841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c00ffae0 0x7f85c00ffef0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f85bc009990 tx=0x7f85bc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 -- 172.21.15.40:0/3888044841 shutdown_connections 2023-07-14T11:11:11.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3888044841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c0101660 0x7f85c0107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3888044841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c00ffae0 0x7f85c00ffef0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3888044841 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 0x7f85c0101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 -- 172.21.15.40:0/3888044841 >> 172.21.15.40:0/3888044841 conn(0x7f85c00fb0e0 msgr2=0x7f85c00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:11.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 -- 172.21.15.40:0/3888044841 shutdown_connections 2023-07-14T11:11:11.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.144+0000 7f85c83ff700 1 -- 172.21.15.40:0/3888044841 wait complete. 2023-07-14T11:11:11.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.145+0000 7f85c83ff700 1 Processor -- start 2023-07-14T11:11:11.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 -- start start 2023-07-14T11:11:11.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 0x7f85c0072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85c00735e0 con 0x7f85c0101660 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85c0073750 con 0x7f85c0100cd0 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c83ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f85c019f910 con 0x7f85c00ffae0 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c619b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c699c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c619b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37186/0 (socket says 172.21.15.40:37186) 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c699c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37394/0 (socket says 172.21.15.40:37394) 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c619b700 1 -- 172.21.15.40:0/3180172445 learned_addr learned my addr 172.21.15.40:0/3180172445 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.146+0000 7f85c599a700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 0x7f85c0072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 -- 172.21.15.40:0/3180172445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 msgr2=0x7f85c0072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 0x7f85c0072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 -- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 msgr2=0x7f85c0072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 -- 172.21.15.40:0/3180172445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f85bc0095f0 con 0x7f85c00ffae0 2023-07-14T11:11:11.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c699c700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:11.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c619b700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f85bc004730 tx=0x7f85bc004760 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f85bc01d070 con 0x7f85c00ffae0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f85c019fb90 con 0x7f85c00ffae0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f85bc021410 con 0x7f85c00ffae0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f85bc017650 con 0x7f85c00ffae0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.147+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f85c019ffb0 con 0x7f85c00ffae0 2023-07-14T11:11:11.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.148+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f85c010b320 con 0x7f85c00ffae0 2023-07-14T11:11:11.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.151+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f85bc021580 con 0x7f85c00ffae0 2023-07-14T11:11:11.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.152+0000 7f85b77fe700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 0x7f85ac0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.152+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f85bc0ce230 con 0x7f85c00ffae0 2023-07-14T11:11:11.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.152+0000 7f85c599a700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 0x7f85ac0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.153+0000 7f85c599a700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 0x7f85ac0a3600 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f85b0009c60 tx=0x7f85b0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:11.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.154+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f85bc097660 con 0x7f85c00ffae0 2023-07-14T11:11:11.365 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.363+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f85c0106a30 con 0x7f85c00ffae0 2023-07-14T11:11:11.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.363+0000 7f85b77fe700 1 -- 172.21.15.40:0/3180172445 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7f85bc09ac80 con 0x7f85c00ffae0 2023-07-14T11:11:11.366 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:11.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 msgr2=0x7f85ac0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 0x7f85ac0a3600 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f85b0009c60 tx=0x7f85b0009260 comp rx=0 tx=0).stop 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 msgr2=0x7f85c00742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f85bc004730 tx=0x7f85bc004760 comp rx=0 tx=0).stop 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 shutdown_connections 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85ac0a1150 0x7f85ac0a3600 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85c0101660 0x7f85c0072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f85c00ffae0 0x7f85c00742c0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 --2- 172.21.15.40:0/3180172445 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f85c0100cd0 0x7f85c0072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 >> 172.21.15.40:0/3180172445 conn(0x7f85c00fb0e0 msgr2=0x7f85c00fd3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:11.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 shutdown_connections 2023-07-14T11:11:11.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.366+0000 7f85c83ff700 1 -- 172.21.15.40:0/3180172445 wait complete. 2023-07-14T11:11:11.388 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:11.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 -- 172.21.15.40:0/545224953 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c10c0b0 msgr2=0x7fbb8c10e490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 --2- 172.21.15.40:0/545224953 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c10e490 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fbb8400b0a0 tx=0x7fbb8400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 -- 172.21.15.40:0/545224953 shutdown_connections 2023-07-14T11:11:11.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 --2- 172.21.15.40:0/545224953 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c10e490 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 --2- 172.21.15.40:0/545224953 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c074550 0x7fbb8c0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 --2- 172.21.15.40:0/545224953 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c072f70 0x7fbb8c0733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 -- 172.21.15.40:0/545224953 >> 172.21.15.40:0/545224953 conn(0x7fbb8c0fb0e0 msgr2=0x7fbb8c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:11.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 -- 172.21.15.40:0/545224953 shutdown_connections 2023-07-14T11:11:11.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.593+0000 7fbb945ff700 1 -- 172.21.15.40:0/545224953 wait complete. 2023-07-14T11:11:11.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.594+0000 7fbb945ff700 1 Processor -- start 2023-07-14T11:11:11.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 -- start start 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c074550 0x7fbb8c19c4d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c1a3540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb8c1a3b10 con 0x7fbb8c10c0b0 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb8c1a3c50 con 0x7fbb8c072f70 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb945ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb8c1a3de0 con 0x7fbb8c074550 2023-07-14T11:11:11.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb9239b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb9239b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35660/0 (socket says 172.21.15.40:35660) 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb9239b700 1 -- 172.21.15.40:0/3751020255 learned_addr learned my addr 172.21.15.40:0/3751020255 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb91b9a700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c074550 0x7fbb8c19c4d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb92b9c700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c1a3540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb9239b700 1 -- 172.21.15.40:0/3751020255 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c074550 msgr2=0x7fbb8c19c4d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.595+0000 7fbb9239b700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c074550 0x7fbb8c19c4d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb9239b700 1 -- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c10c0b0 msgr2=0x7fbb8c1a3540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb9239b700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c1a3540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb9239b700 1 -- 172.21.15.40:0/3751020255 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb84009d00 con 0x7fbb8c072f70 2023-07-14T11:11:11.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb9239b700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800d980 tx=0x7fbb8800dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:11.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8800f890 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb8c1a40c0 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.596+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbb88010460 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.597+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb8c1a4620 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.597+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbb8800e610 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.598+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb70005320 con 0x7fbb8c072f70 2023-07-14T11:11:11.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.600+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fbb880105d0 con 0x7fbb8c072f70 2023-07-14T11:11:11.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.601+0000 7fbb837fe700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 0x7fbb780a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:11.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.601+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fbb880cc970 con 0x7fbb8c072f70 2023-07-14T11:11:11.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.602+0000 7fbb91b9a700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 0x7fbb780a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:11.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.602+0000 7fbb91b9a700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 0x7fbb780a3680 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c074270 tx=0x7fbb7c00b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:11.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.604+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fbb88099820 con 0x7fbb8c072f70 2023-07-14T11:11:11.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.815+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fbb70005f70 con 0x7fbb8c072f70 2023-07-14T11:11:11.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.816+0000 7fbb837fe700 1 -- 172.21.15.40:0/3751020255 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7fbb88017020 con 0x7fbb8c072f70 2023-07-14T11:11:11.819 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:11.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 msgr2=0x7fbb780a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 0x7fbb780a3680 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7fbb8c074270 tx=0x7fbb7c00b410 comp rx=0 tx=0).stop 2023-07-14T11:11:11.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 msgr2=0x7fbb8c19bf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fbb8800d980 tx=0x7fbb8800dc90 comp rx=0 tx=0).stop 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 shutdown_connections 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fbb780a11d0 0x7fbb780a3680 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb8c10c0b0 0x7fbb8c1a3540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.818+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbb8c072f70 0x7fbb8c19bf90 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.819+0000 7fbb945ff700 1 --2- 172.21.15.40:0/3751020255 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbb8c074550 0x7fbb8c19c4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.819+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 >> 172.21.15.40:0/3751020255 conn(0x7fbb8c0fb0e0 msgr2=0x7fbb8c0fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:11.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.819+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 shutdown_connections 2023-07-14T11:11:11.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:11.819+0000 7fbb945ff700 1 -- 172.21.15.40:0/3751020255 wait complete. 2023-07-14T11:11:11.840 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:11 smithi130 ceph-mon[129405]: pgmap v324: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 9 op/s 2023-07-14T11:11:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1433517109' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3180172445' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:11 smithi040 ceph-mon[97954]: pgmap v324: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 9 op/s 2023-07-14T11:11:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1433517109' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3180172445' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 -- 172.21.15.40:0/3905526732 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 msgr2=0x7fae700fff00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 --2- 172.21.15.40:0/3905526732 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae700fff00 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7fae64009990 tx=0x7fae64009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 -- 172.21.15.40:0/3905526732 shutdown_connections 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 --2- 172.21.15.40:0/3905526732 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 0x7fae70107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 --2- 172.21.15.40:0/3905526732 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae700fff00 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 --2- 172.21.15.40:0/3905526732 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 0x7fae70101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 -- 172.21.15.40:0/3905526732 >> 172.21.15.40:0/3905526732 conn(0x7fae700fb130 msgr2=0x7fae700fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:12.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 -- 172.21.15.40:0/3905526732 shutdown_connections 2023-07-14T11:11:12.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.044+0000 7fae76fff700 1 -- 172.21.15.40:0/3905526732 wait complete. 2023-07-14T11:11:12.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 Processor -- start 2023-07-14T11:11:12.049 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 -- start start 2023-07-14T11:11:12.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 0x7fae70195e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 0x7fae7019ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae7019d450 con 0x7fae700ffaf0 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae7019d590 con 0x7fae70100ce0 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.046+0000 7fae76fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae7019d720 con 0x7fae70101670 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37424/0 (socket says 172.21.15.40:37424) 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 -- 172.21.15.40:0/826647914 learned_addr learned my addr 172.21.15.40:0/826647914 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:12.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae6ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 0x7fae70195e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae7559c700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 0x7fae7019ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 -- 172.21.15.40:0/826647914 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 msgr2=0x7fae7019ce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 0x7fae7019ce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 -- 172.21.15.40:0/826647914 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 msgr2=0x7fae70195e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 0x7fae70195e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 -- 172.21.15.40:0/826647914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae640095f0 con 0x7fae700ffaf0 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae74d9b700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fae6400b620 tx=0x7fae64005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:12.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.047+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae6401d070 con 0x7fae700ffaf0 2023-07-14T11:11:12.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.048+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fae64021410 con 0x7fae700ffaf0 2023-07-14T11:11:12.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.048+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae7019d9a0 con 0x7fae700ffaf0 2023-07-14T11:11:12.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.048+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fae64017650 con 0x7fae700ffaf0 2023-07-14T11:11:12.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.048+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae7019ddc0 con 0x7fae700ffaf0 2023-07-14T11:11:12.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.049+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fae70107080 con 0x7fae700ffaf0 2023-07-14T11:11:12.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.050+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fae64021580 con 0x7fae700ffaf0 2023-07-14T11:11:12.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.052+0000 7fae6dffb700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 0x7fae600a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.052+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fae640ce190 con 0x7fae700ffaf0 2023-07-14T11:11:12.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.052+0000 7fae6ffff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 0x7fae600a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.053+0000 7fae6ffff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 0x7fae600a3600 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fae5c005950 tx=0x7fae5c00b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:12.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.056+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fae640975c0 con 0x7fae700ffaf0 2023-07-14T11:11:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:11 smithi045 ceph-mon[128791]: pgmap v324: 129 pgs: 129 active+clean; 586 KiB data, 329 MiB used, 1.0 TiB / 1.0 TiB avail; 5.6 KiB/s rd, 9 op/s 2023-07-14T11:11:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1433517109' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3180172445' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:12.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.267+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7fae70061120 con 0x7fae700ffaf0 2023-07-14T11:11:12.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.267+0000 7fae6dffb700 1 -- 172.21.15.40:0/826647914 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7fae6409abe0 con 0x7fae700ffaf0 2023-07-14T11:11:12.270 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:12.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 msgr2=0x7fae600a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 0x7fae600a3600 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7fae5c005950 tx=0x7fae5c00b410 comp rx=0 tx=0).stop 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 msgr2=0x7fae701958d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7fae6400b620 tx=0x7fae64005740 comp rx=0 tx=0).stop 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 shutdown_connections 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fae600a1150 0x7fae600a3600 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fae70101670 0x7fae7019ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae700ffaf0 0x7fae701958d0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 --2- 172.21.15.40:0/826647914 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fae70100ce0 0x7fae70195e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 >> 172.21.15.40:0/826647914 conn(0x7fae700fb130 msgr2=0x7fae701041c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:12.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 shutdown_connections 2023-07-14T11:11:12.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.270+0000 7fae76fff700 1 -- 172.21.15.40:0/826647914 wait complete. 2023-07-14T11:11:12.291 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:12.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.493+0000 7f5dee5ff700 1 -- 172.21.15.40:0/161727481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8100cf0 msgr2=0x7f5de8101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.493+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/161727481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8100cf0 0x7f5de8101140 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc009990 tx=0x7f5ddc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 -- 172.21.15.40:0/161727481 shutdown_connections 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/161727481 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de8101680 0x7f5de8107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/161727481 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de80ffb00 0x7f5de80fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/161727481 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8100cf0 0x7f5de8101140 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 -- 172.21.15.40:0/161727481 >> 172.21.15.40:0/161727481 conn(0x7f5de80fb0e0 msgr2=0x7f5de80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:12.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 -- 172.21.15.40:0/161727481 shutdown_connections 2023-07-14T11:11:12.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.494+0000 7f5dee5ff700 1 -- 172.21.15.40:0/161727481 wait complete. 2023-07-14T11:11:12.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 Processor -- start 2023-07-14T11:11:12.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 -- start start 2023-07-14T11:11:12.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 0x7f5de8197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5de819f600 con 0x7f5de8101680 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5de819f740 con 0x7f5de80ffb00 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5dee5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5de819f8d0 con 0x7f5de8100cf0 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5decb9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5de7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5de7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35698/0 (socket says 172.21.15.40:35698) 2023-07-14T11:11:12.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5de7fff700 1 -- 172.21.15.40:0/3935785398 learned_addr learned my addr 172.21.15.40:0/3935785398 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.495+0000 7f5decb9c700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37434/0 (socket says 172.21.15.40:37434) 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 0x7f5de8197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 -- 172.21.15.40:0/3935785398 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 msgr2=0x7f5de8197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 -- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 msgr2=0x7f5de819f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 -- 172.21.15.40:0/3935785398 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ddc0095f0 con 0x7f5de8100cf0 2023-07-14T11:11:12.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de7fff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5decb9c700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.496+0000 7f5de77fe700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 0x7f5de8197fc0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc00b620 tx=0x7f5ddc004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.497+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc01d070 con 0x7f5de8100cf0 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.497+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ddc021410 con 0x7f5de8100cf0 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.497+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5de819fb50 con 0x7f5de8100cf0 2023-07-14T11:11:12.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.497+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ddc017650 con 0x7f5de8100cf0 2023-07-14T11:11:12.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.497+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5de819ff70 con 0x7f5de8100cf0 2023-07-14T11:11:12.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.498+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5de810b320 con 0x7f5de8100cf0 2023-07-14T11:11:12.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.500+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5ddc0217f0 con 0x7f5de8100cf0 2023-07-14T11:11:12.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.502+0000 7f5de57fa700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 0x7f5dd00a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:12.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.502+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5ddc0ce720 con 0x7f5de8100cf0 2023-07-14T11:11:12.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.502+0000 7f5de7fff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 0x7f5dd00a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:12.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.502+0000 7f5de7fff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 0x7f5dd00a3650 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f5dd8005950 tx=0x7f5dd8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:12.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.503+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f5ddc097b50 con 0x7f5de8100cf0 2023-07-14T11:11:12.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.758+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f5de81a0640 con 0x7f5de8100cf0 2023-07-14T11:11:12.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.759+0000 7f5de57fa700 1 -- 172.21.15.40:0/3935785398 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f5ddc09b170 con 0x7f5de8100cf0 2023-07-14T11:11:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:12.794 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:12.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 msgr2=0x7f5dd00a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 0x7f5dd00a3650 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f5dd8005950 tx=0x7f5dd8009260 comp rx=0 tx=0).stop 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 msgr2=0x7f5de8197fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 0x7f5de8197fc0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f5ddc00b620 tx=0x7f5ddc004730 comp rx=0 tx=0).stop 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 shutdown_connections 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5dd00a11a0 0x7f5dd00a3650 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5de8101680 0x7f5de819f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5de80ffb00 0x7f5de8197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 --2- 172.21.15.40:0/3935785398 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5de8100cf0 0x7f5de8197fc0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:12.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 >> 172.21.15.40:0/3935785398 conn(0x7f5de80fb0e0 msgr2=0x7f5de8106590 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:12.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 shutdown_connections 2023-07-14T11:11:12.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.762+0000 7f5dee5ff700 1 -- 172.21.15.40:0/3935785398 wait complete. 2023-07-14T11:11:12.797 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:12.798 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 -- 172.21.15.40:0/1092893382 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 msgr2=0x7fd94c101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1092893382 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c101140 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fd940009990 tx=0x7fd940009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 -- 172.21.15.40:0/1092893382 shutdown_connections 2023-07-14T11:11:13.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1092893382 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c101680 0x7fd94c107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1092893382 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1092893382 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c101140 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 -- 172.21.15.40:0/1092893382 >> 172.21.15.40:0/1092893382 conn(0x7fd94c0fb0e0 msgr2=0x7fd94c0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 -- 172.21.15.40:0/1092893382 shutdown_connections 2023-07-14T11:11:13.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.998+0000 7fd9511ff700 1 -- 172.21.15.40:0/1092893382 wait complete. 2023-07-14T11:11:13.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:12.999+0000 7fd9511ff700 1 Processor -- start 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 -- start start 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c101680 0x7fd94c072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c0735e0 con 0x7fd94c100cf0 2023-07-14T11:11:13.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c073750 con 0x7fd94c101680 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.000+0000 7fd9511ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd94c19f7e0 con 0x7fd94c0ffb00 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:37452/0 (socket says 172.21.15.40:37452) 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 -- 172.21.15.40:0/1067938458 learned_addr learned my addr 172.21.15.40:0/1067938458 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94b59e700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c101680 0x7fd94c072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 -- 172.21.15.40:0/1067938458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 msgr2=0x7fd94c0742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 -- 172.21.15.40:0/1067938458 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c101680 msgr2=0x7fd94c072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c101680 0x7fd94c072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 -- 172.21.15.40:0/1067938458 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd9400095f0 con 0x7fd94c100cf0 2023-07-14T11:11:13.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94ad9d700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:13.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd94a59c700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fd94000b620 tx=0x7fd940004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.001+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd94001d070 con 0x7fd94c100cf0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.002+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd94c19fa60 con 0x7fd94c100cf0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.002+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd940005030 con 0x7fd94c100cf0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.002+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd940004b80 con 0x7fd94c100cf0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.002+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd94c19fe80 con 0x7fd94c100cf0 2023-07-14T11:11:13.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.003+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd94c10b340 con 0x7fd94c100cf0 2023-07-14T11:11:13.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.005+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fd9400051a0 con 0x7fd94c100cf0 2023-07-14T11:11:13.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.006+0000 7fd933fff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 0x7fd9340a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.006+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fd9400cdca0 con 0x7fd94c100cf0 2023-07-14T11:11:13.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.006+0000 7fd94ad9d700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 0x7fd9340a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.007+0000 7fd94ad9d700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 0x7fd9340a3650 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fd94c100b50 tx=0x7fd93c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.010+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fd9400970d0 con 0x7fd94c100cf0 2023-07-14T11:11:13.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:12 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3751020255' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:12 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/826647914' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:12 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3751020255' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:12 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/826647914' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:12 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3751020255' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:12 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/826647914' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:13.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.221+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fd94c106980 con 0x7fd94c100cf0 2023-07-14T11:11:13.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.223+0000 7fd933fff700 1 -- 172.21.15.40:0/1067938458 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7fd940004ce0 con 0x7fd94c100cf0 2023-07-14T11:11:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:13.226 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:13.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.225+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 msgr2=0x7fd9340a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.225+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 0x7fd9340a3650 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fd94c100b50 tx=0x7fd93c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 msgr2=0x7fd94c072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fd94000b620 tx=0x7fd940004730 comp rx=0 tx=0).stop 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 shutdown_connections 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd9340a11a0 0x7fd9340a3650 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd94c101680 0x7fd94c072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd94c0ffb00 0x7fd94c0742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 --2- 172.21.15.40:0/1067938458 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd94c100cf0 0x7fd94c072900 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 >> 172.21.15.40:0/1067938458 conn(0x7fd94c0fb0e0 msgr2=0x7fd94c0fd300 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 shutdown_connections 2023-07-14T11:11:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.226+0000 7fd9511ff700 1 -- 172.21.15.40:0/1067938458 wait complete. 2023-07-14T11:11:13.247 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:13.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 -- 172.21.15.40:0/1128842752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898074550 msgr2=0x7f08980729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 --2- 172.21.15.40:0/1128842752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898074550 0x7f08980729a0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f0894009990 tx=0x7f0894009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 -- 172.21.15.40:0/1128842752 shutdown_connections 2023-07-14T11:11:13.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 --2- 172.21.15.40:0/1128842752 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08981079c0 0x7f0898109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 --2- 172.21.15.40:0/1128842752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0898074550 0x7f08980729a0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 --2- 172.21.15.40:0/1128842752 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f08980733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 -- 172.21.15.40:0/1128842752 >> 172.21.15.40:0/1128842752 conn(0x7f08980fb0e0 msgr2=0x7f08980fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 -- 172.21.15.40:0/1128842752 shutdown_connections 2023-07-14T11:11:13.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.449+0000 7f089f5ff700 1 -- 172.21.15.40:0/1128842752 wait complete. 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.450+0000 7f089f5ff700 1 Processor -- start 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 -- start start 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898074550 0x7f0898103360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f08981079c0 0x7f08980ff9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0898100150 con 0x7f0898074550 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08981002c0 con 0x7f08981079c0 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089f5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0898100480 con 0x7f0898072f70 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089d39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089cb9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898074550 0x7f0898103360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089d39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37252/0 (socket says 172.21.15.40:37252) 2023-07-14T11:11:13.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.451+0000 7f089d39b700 1 -- 172.21.15.40:0/997037790 learned_addr learned my addr 172.21.15.40:0/997037790 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089db9c700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f08981079c0 0x7f08980ff9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 -- 172.21.15.40:0/997037790 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f08981079c0 msgr2=0x7f08980ff9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f08981079c0 0x7f08980ff9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 -- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898074550 msgr2=0x7f0898103360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898074550 0x7f0898103360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 -- 172.21.15.40:0/997037790 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08940095f0 con 0x7f0898072f70 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089d39b700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f0894000c00 tx=0x7f0894005720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f089401d070 con 0x7f0898072f70 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0894022470 con 0x7f0898072f70 2023-07-14T11:11:13.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08981a3f60 con 0x7f0898072f70 2023-07-14T11:11:13.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.452+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0894017650 con 0x7f0898072f70 2023-07-14T11:11:13.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.453+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08981a42d0 con 0x7f0898072f70 2023-07-14T11:11:13.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.453+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0898106070 con 0x7f0898072f70 2023-07-14T11:11:13.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.456+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f08940225e0 con 0x7f0898072f70 2023-07-14T11:11:13.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.457+0000 7f088e7fc700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 0x7f08840a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.457+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f08940ce510 con 0x7f0898072f70 2023-07-14T11:11:13.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.457+0000 7f089cb9a700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 0x7f08840a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.458+0000 7f089cb9a700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 0x7f08840a3650 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0898074270 tx=0x7f0888009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.459+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0894097940 con 0x7f0898072f70 2023-07-14T11:11:13.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.717+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f089802ca90 con 0x7f0898072f70 2023-07-14T11:11:13.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.718+0000 7f088e7fc700 1 -- 172.21.15.40:0/997037790 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f089409af60 con 0x7f0898072f70 2023-07-14T11:11:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:13.721 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 msgr2=0x7f08840a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 0x7f08840a3650 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0898074270 tx=0x7f0888009260 comp rx=0 tx=0).stop 2023-07-14T11:11:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 msgr2=0x7f0898102e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f0894000c00 tx=0x7f0894005720 comp rx=0 tx=0).stop 2023-07-14T11:11:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 shutdown_connections 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f08840a11a0 0x7f08840a3650 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f08981079c0 0x7f08980ff9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0898072f70 0x7f0898102e20 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 --2- 172.21.15.40:0/997037790 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0898074550 0x7f0898103360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 >> 172.21.15.40:0/997037790 conn(0x7f08980fb0e0 msgr2=0x7f08980fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 shutdown_connections 2023-07-14T11:11:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.720+0000 7f089f5ff700 1 -- 172.21.15.40:0/997037790 wait complete. 2023-07-14T11:11:13.725 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:13.742 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 27, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2023-07-14T11:09:43.127642+0000', 'modified': '2023-07-14T11:10:17.374269+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 5, 'in': [0, 1, 2, 3, 4], 'up': {'mds_0': 24569, 'mds_1': 24565, 'mds_2': 24523, 'mds_3': 24521, 'mds_4': 24511}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24466': {'gid': 24466, 'name': 'l', 'rank': 4, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6833/2180852091', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6832', 'nonce': 2180852091}, {'type': 'v1', 'addr': '172.21.15.130:6833', 'nonce': 2180852091}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24482': {'gid': 24482, 'name': 'd', 'rank': 3, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.40:6837/625677299', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6836', 'nonce': 625677299}, {'type': 'v1', 'addr': '172.21.15.40:6837', 'nonce': 625677299}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24494': {'gid': 24494, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.45:6835/1717726567', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6834', 'nonce': 1717726567}, {'type': 'v1', 'addr': '172.21.15.45:6835', 'nonce': 1717726567}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24511': {'gid': 24511, 'name': 'f', 'rank': 4, 'incarnation': 25, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.130:6835/2228846875', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6834', 'nonce': 2228846875}, {'type': 'v1', 'addr': '172.21.15.130:6835', 'nonce': 2228846875}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24521': {'gid': 24521, 'name': 'h', 'rank': 3, 'incarnation': 23, 'state': 'up:active', 'state_seq': 14, 'addr': '172.21.15.45:6837/3949707739', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6836', 'nonce': 3949707739}, {'type': 'v1', 'addr': '172.21.15.45:6837', 'nonce': 3949707739}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24523': {'gid': 24523, 'name': 'c', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 14, 'addr': '172.21.15.130:6837/4266211241', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6836', 'nonce': 4266211241}, {'type': 'v1', 'addr': '172.21.15.130:6837', 'nonce': 4266211241}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24541': {'gid': 24541, 'name': 'i', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6839/3942717247', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6838', 'nonce': 3942717247}, {'type': 'v1', 'addr': '172.21.15.130:6839', 'nonce': 3942717247}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24553': {'gid': 24553, 'name': 'g', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.40:6839/3591632989', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6838', 'nonce': 3591632989}, {'type': 'v1', 'addr': '172.21.15.40:6839', 'nonce': 3591632989}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24565': {'gid': 24565, 'name': 'a', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 11, 'addr': '172.21.15.40:6841/1989097261', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6840', 'nonce': 1989097261}, {'type': 'v1', 'addr': '172.21.15.40:6841', 'nonce': 1989097261}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24569': {'gid': 24569, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.45:6839/180621035', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6838', 'nonce': 180621035}, {'type': 'v1', 'addr': '172.21.15.45:6839', 'nonce': 180621035}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0} 2023-07-14T11:11:13.742 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 5/5 2023-07-14T11:11:13.743 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.f status 2023-07-14T11:11:13.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 -- 172.21.15.40:0/3545461772 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c101660 msgr2=0x7fa25c107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3545461772 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c101660 0x7fa25c107f20 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fa25400b0a0 tx=0x7fa25400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 -- 172.21.15.40:0/3545461772 shutdown_connections 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3545461772 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c101660 0x7fa25c107f20 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3545461772 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.943+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3545461772 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c100cd0 0x7fa25c101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.944+0000 7fa2617ff700 1 -- 172.21.15.40:0/3545461772 >> 172.21.15.40:0/3545461772 conn(0x7fa25c0fb0e0 msgr2=0x7fa25c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.944+0000 7fa2617ff700 1 -- 172.21.15.40:0/3545461772 shutdown_connections 2023-07-14T11:11:13.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.944+0000 7fa2617ff700 1 -- 172.21.15.40:0/3545461772 wait complete. 2023-07-14T11:11:13.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 Processor -- start 2023-07-14T11:11:13.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 -- start start 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c100cd0 0x7fa25c198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c101660 0x7fa25c19f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa25c19f6d0 con 0x7fa25c0ffae0 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa25c19f810 con 0x7fa25c100cd0 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa2617ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa25c19f9a0 con 0x7fa25c101660 2023-07-14T11:11:13.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa25affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa25affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42322/0 (socket says 172.21.15.40:42322) 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa25affd700 1 -- 172.21.15.40:0/3258572064 learned_addr learned my addr 172.21.15.40:0/3258572064 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c100cd0 0x7fa25c198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.945+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c101660 0x7fa25c19f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 -- 172.21.15.40:0/3258572064 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c101660 msgr2=0x7fa25c19f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c101660 0x7fa25c19f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 -- 172.21.15.40:0/3258572064 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c100cd0 msgr2=0x7fa25c198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c100cd0 0x7fa25c198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 -- 172.21.15.40:0/3258572064 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa254009d00 con 0x7fa25c0ffae0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa25affd700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fa24c00e970 tx=0x7fa24c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa243fff700 1 -- 172.21.15.40:0/3258572064 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa24c009790 con 0x7fa25c0ffae0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa243fff700 1 -- 172.21.15.40:0/3258572064 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa24c00c7e0 con 0x7fa25c0ffae0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa25c19fc80 con 0x7fa25c0ffae0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa243fff700 1 -- 172.21.15.40:0/3258572064 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa24c003c70 con 0x7fa25c0ffae0 2023-07-14T11:11:13.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.946+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa25c1a0190 con 0x7fa25c0ffae0 2023-07-14T11:11:13.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.951+0000 7fa243fff700 1 -- 172.21.15.40:0/3258572064 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fa24c00cc50 con 0x7fa25c0ffae0 2023-07-14T11:11:13.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.952+0000 7fa275037b80 1 Processor -- start 2023-07-14T11:11:13.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.952+0000 7fa275037b80 1 -- start start 2023-07-14T11:11:13.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.952+0000 7fa275037b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e14f890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa275037b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa275037b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e151090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa275037b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e054c60 con 0x55565e150c20 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa275037b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e0a79f0 con 0x55565e14f480 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa275037b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e0a0e80 con 0x55565e14fdd0 2023-07-14T11:11:13.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa25affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa25affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50608/0 (socket says 172.21.15.40:50608) 2023-07-14T11:11:13.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa25affd700 1 -- 172.21.15.40:0/2048459676 learned_addr learned my addr 172.21.15.40:0/2048459676 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e14f890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.953+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e151090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa243fff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 0x7fa2440a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 -- 172.21.15.40:0/2048459676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 msgr2=0x55565e14f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e14f890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 -- 172.21.15.40:0/2048459676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 msgr2=0x55565e151090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e151090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 -- 172.21.15.40:0/2048459676 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55565deced90 con 0x55565e14fdd0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa243fff700 1 -- 172.21.15.40:0/3258572064 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fa24c014070 con 0x7fa25c0ffae0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 0x7fa2440a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa25affd700 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fa24c065a10 tx=0x7fa24c065300 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e6118ebbfd213479 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa2417fa700 1 -- 172.21.15.40:0/2048459676 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa24c099c30 con 0x55565e14fdd0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa2417fa700 1 -- 172.21.15.40:0/2048459676 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa24c0999a0 con 0x55565e14fdd0 2023-07-14T11:11:13.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.954+0000 7fa2417fa700 1 -- 172.21.15.40:0/2048459676 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa24c065370 con 0x55565e14fdd0 2023-07-14T11:11:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 0x7fa2440a3680 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fa25400b780 tx=0x7fa2540099e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 -- 172.21.15.40:0/2048459676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 msgr2=0x55565e150690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fa24c065a10 tx=0x7fa24c065300 comp rx=0 tx=0).stop 2023-07-14T11:11:13.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 -- 172.21.15.40:0/2048459676 shutdown_connections 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e151090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e14f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 --2- 172.21.15.40:0/2048459676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e150690 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 -- 172.21.15.40:0/2048459676 >> 172.21.15.40:0/2048459676 conn(0x55565e144d50 msgr2=0x55565e14a180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 -- 172.21.15.40:0/2048459676 shutdown_connections 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.955+0000 7fa275037b80 1 -- 172.21.15.40:0/2048459676 wait complete. 2023-07-14T11:11:13.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.956+0000 7fa275037b80 1 -- start start 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.956+0000 7fa275037b80 1 Processor -- start 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.956+0000 7fa275037b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.956+0000 7fa275037b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 0x55565e161dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.956+0000 7fa275037b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa275037b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e040a70 con 0x55565e150c20 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa275037b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e040770 con 0x55565e14fdd0 2023-07-14T11:11:13.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa275037b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565df248d0 con 0x55565e14f480 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa275037b80 10 client.0 _ll_register_callbacks cb 0x55565e03e710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25a7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 0x55565e161dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50618/0 (socket says 172.21.15.40:50618) 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25a7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 0x55565e161dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60936/0 (socket says 172.21.15.40:60936) 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 -- 172.21.15.40:0/4244077803 learned_addr learned my addr 172.21.15.40:0/4244077803 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42340/0 (socket says 172.21.15.40:42340) 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 -- 172.21.15.40:0/4244077803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 msgr2=0x55565e161dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 0x55565e161dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 -- 172.21.15.40:0/4244077803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 msgr2=0x55565e168e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25affd700 1 -- 172.21.15.40:0/4244077803 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa25c1a0190 con 0x55565e14f480 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.957+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa25affd700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fa24c099df0 tx=0x7fa24c000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=66317773da6e52f4 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa24c099c30 con 0x55565e14f480 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fa24c065370 con 0x55565e14f480 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55565dffa4a0 con 0x55565e14f480 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa24c0997c0 con 0x55565e14f480 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa275037b80 10 client.25114 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55565deb7c10 con 0x55565e14f480 2023-07-14T11:11:13.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55565dffa4a0 con 0x55565e14f480 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fa24c00c950 con 0x55565e14f480 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.958+0000 7fa275037b80 10 client.25114 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa275037b80 10 client.25114 resolve_mds: resolved name 'f' to daemon mds.f 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa275037b80 20 client.25114 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e16c710 0x55565e16bd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa275037b80 4 client.25114 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:13.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55565e099e60 con 0x55565e16c710 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e16c710 0x55565e16bd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e16c710 0x55565e16bd30 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=e8b357f109db7f7e server_cookie=721dcac63b61ee2e in_seq=0 out_seq=0 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.959+0000 7fa23eff5700 10 client.25114 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.960+0000 7fa23eff5700 1 -- 172.21.15.40:0/4244077803 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55565e099e60 con 0x55565e16c710 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.960+0000 7fa23eff5700 10 client.25114 handle_command_reply: tid=0 2023-07-14T11:11:13.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.960+0000 7fa275037b80 1 client.25114 shutdown 2023-07-14T11:11:13.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.960+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 msgr2=0x55565e161880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.960+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7fa24c099df0 tx=0x7fa24c000bc0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 shutdown_connections 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e16c710 0x55565e16bd30 crc :-1 s=READY pgs=18 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e168e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa23eff5700 0 client.25114 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e161880 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 --2- 172.21.15.40:0/4244077803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14fdd0 0x55565e161dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 >> 172.21.15.40:0/4244077803 conn(0x55565e144d50 msgr2=0x55565e149860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 shutdown_connections 2023-07-14T11:11:13.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 1 -- 172.21.15.40:0/4244077803 wait complete. 2023-07-14T11:11:13.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.961+0000 7fa275037b80 20 client.25114 trim_cache size 0 max 16384 2023-07-14T11:11:13.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.979+0000 7fa275037b80 1 Processor -- start 2023-07-14T11:11:13.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 -- start start 2023-07-14T11:11:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e16bd30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e168dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e162780 con 0x55565e150c20 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e144990 con 0x55565e14f480 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa275037b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e099e60 con 0x55565e14fdd0 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42356/0 (socket says 172.21.15.40:42356) 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 -- 172.21.15.40:0/955512577 learned_addr learned my addr 172.21.15.40:0/955512577 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25affd700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e16bd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e168dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 -- 172.21.15.40:0/955512577 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 msgr2=0x55565e168dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e168dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 -- 172.21.15.40:0/955512577 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 msgr2=0x55565e16bd30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e16bd30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25b7fe700 1 -- 172.21.15.40:0/955512577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55565def5200 con 0x55565e150c20 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.980+0000 7fa25affd700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e16bd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.981+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fa25400b0a0 tx=0x7fa25401e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=136013bcc3dfc129 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.981+0000 7fa241ffb700 1 -- 172.21.15.40:0/955512577 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa25400e040 con 0x55565e150c20 2023-07-14T11:11:13.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.981+0000 7fa241ffb700 1 -- 172.21.15.40:0/955512577 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa25400cde0 con 0x55565e150c20 2023-07-14T11:11:13.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.981+0000 7fa241ffb700 1 -- 172.21.15.40:0/955512577 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa25401ee40 con 0x55565e150c20 2023-07-14T11:11:13.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- 172.21.15.40:0/955512577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 msgr2=0x55565e157060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fa25400b0a0 tx=0x7fa25401e000 comp rx=0 tx=0).stop 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- 172.21.15.40:0/955512577 shutdown_connections 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e150c20 0x55565e157060 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e14f480 0x55565e16bd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 --2- 172.21.15.40:0/955512577 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14fdd0 0x55565e168dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- 172.21.15.40:0/955512577 >> 172.21.15.40:0/955512577 conn(0x55565e144d50 msgr2=0x55565e1605a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- 172.21.15.40:0/955512577 shutdown_connections 2023-07-14T11:11:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- 172.21.15.40:0/955512577 wait complete. 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 -- start start 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.982+0000 7fa275037b80 1 Processor -- start 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e1528b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 0x55565e167300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e167840 con 0x55565e14fdd0 2023-07-14T11:11:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e1679b0 con 0x55565e150c20 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565e167b70 con 0x55565e14f480 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa275037b80 10 client.0 _ll_register_callbacks cb 0x7fa250009830 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42370/0 (socket says 172.21.15.40:42370) 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 -- 172.21.15.40:0/106167886 learned_addr learned my addr 172.21.15.40:0/106167886 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25affd700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e1528b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 0x55565e167300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 -- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 msgr2=0x55565e1528b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e1528b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 -- 172.21.15.40:0/106167886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 msgr2=0x55565e167300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 0x55565e167300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25a7fc700 1 -- 172.21.15.40:0/106167886 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa25c108850 con 0x55565e14fdd0 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.983+0000 7fa25b7fe700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 0x55565e167300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa25a7fc700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x55565e1628e0 tx=0x7fa250010870 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fdbf52efc0c29fcb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa23eff5700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa250017040 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa24c0999a0 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fa25001ebd0 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa25001e600 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa275037b80 10 client.15108 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.984+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55565decd260 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa24c0999a0 con 0x55565e14fdd0 2023-07-14T11:11:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fa250024020 con 0x55565e14fdd0 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 10 client.15108 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 10 client.15108 resolve_mds: resolved name 'f' to daemon mds.f 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 20 client.15108 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e1c5bb0 0x55565e1c2580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 4 client.15108 mds_command: new command op to 24511 tid=0[{"prefix": "status"}] 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "status"}) v1 -- 0x55565e05c000 con 0x55565e1c5bb0 2023-07-14T11:11:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.985+0000 7fa25affd700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e1c5bb0 0x55565e1c2580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa25affd700 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e1c5bb0 0x55565e1c2580 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=83dbb65a0bd516d0 server_cookie=6b690926cc0d0f0c in_seq=0 out_seq=0 2023-07-14T11:11:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa23eff5700 10 client.15108 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa23eff5700 1 -- 172.21.15.40:0/106167886 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+326 (crc 0 0 0) 0x55565e05c000 con 0x55565e1c5bb0 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa23eff5700 10 client.15108 handle_command_reply: tid=0 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 client.15108 shutdown 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 msgr2=0x55565e152df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x55565e1628e0 tx=0x7fa250010870 comp rx=0 tx=0).stop 2023-07-14T11:11:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 shutdown_connections 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55565e1c5bb0 0x55565e1c2580 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565e150c20 0x55565e167300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565e14f480 0x55565e1528b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa23eff5700 0 client.15108 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 --2- 172.21.15.40:0/106167886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565e14fdd0 0x55565e152df0 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 >> 172.21.15.40:0/106167886 conn(0x55565e144d50 msgr2=0x55565e16bd30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 shutdown_connections 2023-07-14T11:11:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 1 -- 172.21.15.40:0/106167886 wait complete. 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.986+0000 7fa275037b80 20 client.15108 trim_cache size 0 max 16384 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 4, 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24511, 2023-07-14T11:11:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 57.621413206, 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 111.09413960000001 2023-07-14T11:11:14.000 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 msgr2=0x7fa2440a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 0x7fa2440a3680 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7fa25400b780 tx=0x7fa2540099e0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 msgr2=0x7fa25c197b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fa24c00e970 tx=0x7fa24c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 shutdown_connections 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fa2440a11d0 0x7fa2440a3680 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa25c101660 0x7fa25c19f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa25c0ffae0 0x7fa25c197b50 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 --2- 172.21.15.40:0/3258572064 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa25c100cd0 0x7fa25c198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 >> 172.21.15.40:0/3258572064 conn(0x7fa25c0fb0e0 msgr2=0x7fa25c0fd280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 shutdown_connections 2023-07-14T11:11:14.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:13.987+0000 7fa2617ff700 1 -- 172.21.15.40:0/3258572064 wait complete. 2023-07-14T11:11:14.005 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:13 smithi040 ceph-mon[97954]: pgmap v325: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 9 op/s 2023-07-14T11:11:14.006 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:13 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3935785398' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.006 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:13 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1067938458' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.006 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:13 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/997037790' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.008 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2023-07-14T11:11:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:13 smithi130 ceph-mon[129405]: pgmap v325: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 9 op/s 2023-07-14T11:11:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:13 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3935785398' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:13 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1067938458' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:13 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/997037790' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:13 smithi045 ceph-mon[128791]: pgmap v325: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 3.7 KiB/s rd, 9 op/s 2023-07-14T11:11:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:13 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3935785398' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:13 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1067938458' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:13 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/997037790' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:14.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 -- 172.21.15.40:0/48386592 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c100cd0 msgr2=0x7fcb0c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 --2- 172.21.15.40:0/48386592 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c101120 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7fcaf4009990 tx=0x7fcaf4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 -- 172.21.15.40:0/48386592 shutdown_connections 2023-07-14T11:11:14.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 --2- 172.21.15.40:0/48386592 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 0x7fcb0c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 --2- 172.21.15.40:0/48386592 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 --2- 172.21.15.40:0/48386592 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c101120 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 -- 172.21.15.40:0/48386592 >> 172.21.15.40:0/48386592 conn(0x7fcb0c0fb0e0 msgr2=0x7fcb0c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 -- 172.21.15.40:0/48386592 shutdown_connections 2023-07-14T11:11:14.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.208+0000 7fcb10bff700 1 -- 172.21.15.40:0/48386592 wait complete. 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.209+0000 7fcb10bff700 1 Processor -- start 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 -- start start 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 0x7fcb0c072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0c0735e0 con 0x7fcb0c0ffae0 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0c073750 con 0x7fcb0c100cd0 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb10bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0c19f8c0 con 0x7fcb0c101660 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60970/0 (socket says 172.21.15.40:60970) 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb03fff700 1 -- 172.21.15.40:0/1331348371 learned_addr learned my addr 172.21.15.40:0/1331348371 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.210+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 0x7fcb0c072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 -- 172.21.15.40:0/1331348371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 msgr2=0x7fcb0c072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 0x7fcb0c072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 -- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 msgr2=0x7fcb0c0742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 -- 172.21.15.40:0/1331348371 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcaf40095f0 con 0x7fcb0c100cd0 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb03fff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fcaf4004760 tx=0x7fcaf4004790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb037fe700 1 -- 172.21.15.40:0/1331348371 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf401d070 con 0x7fcb0c100cd0 2023-07-14T11:11:14.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcb0c19fb40 con 0x7fcb0c100cd0 2023-07-14T11:11:14.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb037fe700 1 -- 172.21.15.40:0/1331348371 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcaf4004830 con 0x7fcb0c100cd0 2023-07-14T11:11:14.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb037fe700 1 -- 172.21.15.40:0/1331348371 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf4005350 con 0x7fcb0c100cd0 2023-07-14T11:11:14.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.211+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcb0c19ff00 con 0x7fcb0c100cd0 2023-07-14T11:11:14.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.215+0000 7fcb037fe700 1 -- 172.21.15.40:0/1331348371 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fcaf4005510 con 0x7fcb0c100cd0 2023-07-14T11:11:14.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 Processor -- start 2023-07-14T11:11:14.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 -- start start 2023-07-14T11:11:14.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x5654946e7890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946e8690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 0x5654946e9090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654945ecc60 con 0x5654946e8c20 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56549463f9f0 con 0x5654946e7dd0 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x565494638e80 con 0x5654946e7480 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb037fe700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 0x7fcaf80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb0ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x5654946e7890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb0ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x5654946e7890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50662/0 (socket says 172.21.15.40:50662) 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 0x5654946e9090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.216+0000 7fcb0ad9d700 1 -- 172.21.15.40:0/3181966096 learned_addr learned my addr 172.21.15.40:0/3181966096 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946e8690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb037fe700 1 -- 172.21.15.40:0/1331348371 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fcaf40cde60 con 0x7fcb0c100cd0 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 0x7fcaf80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 -- 172.21.15.40:0/3181966096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 msgr2=0x5654946e7890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x5654946e7890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 -- 172.21.15.40:0/3181966096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 msgr2=0x5654946e8690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946e8690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 -- 172.21.15.40:0/3181966096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x565494466d90 con 0x5654946e8c20 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 0x5654946e9090 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fcaf40cb550 tx=0x7fcaf4009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be48dc3ac9c4aa6e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.217+0000 7fcb00ff9700 1 -- 172.21.15.40:0/3181966096 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf409aa90 con 0x5654946e8c20 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 0x7fcaf80a3600 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fcb040060b0 tx=0x7fcb04004b40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb00ff9700 1 -- 172.21.15.40:0/3181966096 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcaf409a8b0 con 0x5654946e8c20 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb00ff9700 1 -- 172.21.15.40:0/3181966096 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf409a440 con 0x5654946e8c20 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3181966096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 msgr2=0x5654946e9090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 0x5654946e9090 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7fcaf40cb550 tx=0x7fcaf4009310 comp rx=0 tx=0).stop 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3181966096 shutdown_connections 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e8c20 0x5654946e9090 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x5654946e7890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3181966096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946e8690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3181966096 >> 172.21.15.40:0/3181966096 conn(0x5654946dcd50 msgr2=0x5654946e2180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3181966096 shutdown_connections 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.218+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3181966096 wait complete. 2023-07-14T11:11:14.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.219+0000 7fcb2443eb80 1 -- start start 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.219+0000 7fcb2443eb80 1 Processor -- start 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.219+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 0x5654946f9aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e8c20 0x565494700ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654945d8a70 con 0x5654946e7dd0 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654945d8770 con 0x5654946e7480 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654944bc8d0 con 0x5654946e8c20 2023-07-14T11:11:14.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb2443eb80 10 client.0 _ll_register_callbacks cb 0x5654945d6710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 0x5654946f9aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60982/0 (socket says 172.21.15.40:60982) 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 -- 172.21.15.40:0/3091454676 learned_addr learned my addr 172.21.15.40:0/3091454676 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e8c20 0x565494700ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 -- 172.21.15.40:0/3091454676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e8c20 msgr2=0x565494700ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e8c20 0x565494700ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 -- 172.21.15.40:0/3091454676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 msgr2=0x5654946f9aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 0x5654946f9aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb0a59c700 1 -- 172.21.15.40:0/3091454676 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcb0c19ff00 con 0x5654946e7480 2023-07-14T11:11:14.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.220+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 0x5654946f9aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fcafc012d20 tx=0x7fcafc00f6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c7c52f26a623b208 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcaf1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:11:14.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcafc017070 con 0x5654946e7480 2023-07-14T11:11:14.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5654945924a0 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fcafc00fc30 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcafc00fe30 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcb2443eb80 10 client.25115 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x56549444fc10 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.221+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5654945924a0 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fcafc003a40 con 0x5654946e7480 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 10 client.25115 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 10 client.25115 resolve_mds: resolved name 'h' to daemon mds.h 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 20 client.25115 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x565494704840 0x565494703c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 4 client.25115 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x565494631e60 con 0x565494704840 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.222+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x565494704840 0x565494703c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.223+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x565494704840 0x565494703c40 crc :-1 s=READY pgs=23 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=b912eb1a2dca153 server_cookie=724b7cb3aaa3a3de in_seq=0 out_seq=0 2023-07-14T11:11:14.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.223+0000 7fcaf1ffb700 10 client.25115 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.223+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3091454676 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x565494631e60 con 0x565494704840 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.223+0000 7fcaf1ffb700 10 client.25115 handle_command_reply: tid=0 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.223+0000 7fcb2443eb80 1 client.25115 shutdown 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 msgr2=0x5654946f9560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7fcafc012d20 tx=0x7fcafc00f6a0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 shutdown_connections 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x565494704840 0x565494703c40 crc :-1 s=READY pgs=23 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e8c20 0x565494700ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7480 0x5654946f9560 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcaf1ffb700 0 client.25115 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3091454676 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7dd0 0x5654946f9aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 >> 172.21.15.40:0/3091454676 conn(0x5654946dcd50 msgr2=0x5654946e1860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 shutdown_connections 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3091454676 wait complete. 2023-07-14T11:11:14.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.224+0000 7fcb2443eb80 20 client.25115 trim_cache size 0 max 16384 2023-07-14T11:11:14.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 Processor -- start 2023-07-14T11:11:14.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 -- start start 2023-07-14T11:11:14.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x565494704180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x565494700aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654945924a0 con 0x5654946dcd50 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654946e8520 con 0x5654946e7dd0 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x565494607790 con 0x5654946e7480 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42428/0 (socket says 172.21.15.40:42428) 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb0a59c700 1 -- 172.21.15.40:0/1047758925 learned_addr learned my addr 172.21.15.40:0/1047758925 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb03fff700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x565494704180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.243+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x565494700aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 -- 172.21.15.40:0/1047758925 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 msgr2=0x565494704180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x565494704180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 -- 172.21.15.40:0/1047758925 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 msgr2=0x565494700aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x565494700aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 -- 172.21.15.40:0/1047758925 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56549448d200 con 0x5654946dcd50 2023-07-14T11:11:14.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb03fff700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x565494704180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fcafc0094b0 tx=0x7fcafc012320 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fad4c6b263329a86 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb017fa700 1 -- 172.21.15.40:0/1047758925 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcafc017070 con 0x5654946dcd50 2023-07-14T11:11:14.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb017fa700 1 -- 172.21.15.40:0/1047758925 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcafc010500 con 0x5654946dcd50 2023-07-14T11:11:14.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.244+0000 7fcb017fa700 1 -- 172.21.15.40:0/1047758925 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcafc010760 con 0x5654946dcd50 2023-07-14T11:11:14.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- 172.21.15.40:0/1047758925 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 msgr2=0x565494703c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7fcafc0094b0 tx=0x7fcafc012320 comp rx=0 tx=0).stop 2023-07-14T11:11:14.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- 172.21.15.40:0/1047758925 shutdown_connections 2023-07-14T11:11:14.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x565494700aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946dcd50 0x565494703c40 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/1047758925 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946e7480 0x565494704180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- 172.21.15.40:0/1047758925 >> 172.21.15.40:0/1047758925 conn(0x5654946efe00 msgr2=0x5654946eef50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- 172.21.15.40:0/1047758925 shutdown_connections 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- 172.21.15.40:0/1047758925 wait complete. 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.245+0000 7fcb2443eb80 1 -- start start 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 Processor -- start 2023-07-14T11:11:14.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 0x5654946fc340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946ff7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654946ffd00 con 0x5654946e7480 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5654946ffe70 con 0x5654946e7dd0 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x565494700030 con 0x5654946dcd50 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.246+0000 7fcb2443eb80 10 client.0 _ll_register_callbacks cb 0x7fcafc0127f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 0x5654946fc340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42444/0 (socket says 172.21.15.40:42444) 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 -- 172.21.15.40:0/3331377281 learned_addr learned my addr 172.21.15.40:0/3331377281 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb0a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 0x5654946fc340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50688/0 (socket says 172.21.15.40:50688) 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb0ad9d700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946ff7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 -- 172.21.15.40:0/3331377281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 msgr2=0x5654946fc340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 0x5654946fc340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 -- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 msgr2=0x5654946ff7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946ff7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 -- 172.21.15.40:0/3331377281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x565494466d90 con 0x5654946e7480 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb03fff700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fcaf409aee0 tx=0x7fcaf400f840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d5636bd30eed8f5d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcaf1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf40a3310 con 0x5654946e7480 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fcaf409a8b0 con 0x5654946e7480 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.247+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fcaf40992f0 con 0x5654946e7480 2023-07-14T11:11:14.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcaf40c0080 con 0x5654946e7480 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 10 client.15126 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7fcafc021040 con 0x5654946e7480 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcaf40b7020 con 0x5654946e7480 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fcaf40a2e80 con 0x5654946e7480 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 10 client.15126 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 10 client.15126 resolve_mds: resolved name 'h' to daemon mds.h 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 20 client.15126 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5654946e8c20 0x56549475a910 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 4 client.15126 mds_command: new command op to 24521 tid=0[{"prefix": "status"}] 2023-07-14T11:11:14.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.248+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5654945f4000 con 0x5654946e8c20 2023-07-14T11:11:14.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5654946e8c20 0x56549475a910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcb0a59c700 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5654946e8c20 0x56549475a910 crc :-1 s=READY pgs=24 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=e6afe9e5338d89f5 server_cookie=963d9e0df5b33fb3 in_seq=0 out_seq=0 2023-07-14T11:11:14.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcaf1ffb700 10 client.15126 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:14.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcaf1ffb700 1 -- 172.21.15.40:0/3331377281 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+320 (crc 0 0 0) 0x5654945f4000 con 0x5654946e8c20 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcaf1ffb700 10 client.15126 handle_command_reply: tid=0 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcb2443eb80 1 client.15126 shutdown 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 msgr2=0x5654946fc880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.249+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7fcaf409aee0 tx=0x7fcaf400f840 comp rx=0 tx=0).stop 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 shutdown_connections 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5654946e8c20 0x56549475a910 crc :-1 s=READY pgs=24 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5654946e7dd0 0x5654946ff7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5654946dcd50 0x5654946fc340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcaf1ffb700 0 client.15126 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 --2- 172.21.15.40:0/3331377281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5654946e7480 0x5654946fc880 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 >> 172.21.15.40:0/3331377281 conn(0x5654946efe00 msgr2=0x565494703c40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 shutdown_connections 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 1 -- 172.21.15.40:0/3331377281 wait complete. 2023-07-14T11:11:14.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb2443eb80 20 client.15126 trim_cache size 0 max 16384 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 3, 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24521, 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 58.892552152, 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 106.94870457 2023-07-14T11:11:14.263 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:14.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 msgr2=0x7fcaf80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 0x7fcaf80a3600 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7fcb040060b0 tx=0x7fcb04004b40 comp rx=0 tx=0).stop 2023-07-14T11:11:14.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 msgr2=0x7fcb0c072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.250+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fcaf4004760 tx=0x7fcaf4004790 comp rx=0 tx=0).stop 2023-07-14T11:11:14.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 shutdown_connections 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcaf80a1150 0x7fcaf80a3600 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb0c101660 0x7fcb0c072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb0c0ffae0 0x7fcb0c0742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 --2- 172.21.15.40:0/1331348371 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb0c100cd0 0x7fcb0c072900 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 >> 172.21.15.40:0/1331348371 conn(0x7fcb0c0fb0e0 msgr2=0x7fcb0c0fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 shutdown_connections 2023-07-14T11:11:14.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.251+0000 7fcb10bff700 1 -- 172.21.15.40:0/1331348371 wait complete. 2023-07-14T11:11:14.271 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/934090714 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94100cd0 msgr2=0x7f7c94101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/934090714 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94100cd0 0x7f7c94101120 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7c840098f0 tx=0x7f7c84009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/934090714 shutdown_connections 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/934090714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94101660 0x7f7c94107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/934090714 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c940ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/934090714 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94100cd0 0x7f7c94101120 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/934090714 >> 172.21.15.40:0/934090714 conn(0x7f7c940fb0e0 msgr2=0x7f7c940fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/934090714 shutdown_connections 2023-07-14T11:11:14.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.475+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/934090714 wait complete. 2023-07-14T11:11:14.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.476+0000 7f7c9c1ff700 1 Processor -- start 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.476+0000 7f7c9c1ff700 1 -- start start 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.476+0000 7f7c9c1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c94197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9c1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9c1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 0x7f7c9419f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9c1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c9419f600 con 0x7f7c94100cd0 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9c1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c9419f740 con 0x7f7c940ffae0 2023-07-14T11:11:14.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9c1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c9419f8d0 con 0x7f7c94101660 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c94197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42468/0 (socket says 172.21.15.40:42468) 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 -- 172.21.15.40:0/2862346873 learned_addr learned my addr 172.21.15.40:0/2862346873 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 0x7f7c9419f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 -- 172.21.15.40:0/2862346873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 msgr2=0x7f7c9419f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 0x7f7c9419f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 -- 172.21.15.40:0/2862346873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 msgr2=0x7f7c94197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c94197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9979a700 1 -- 172.21.15.40:0/2862346873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c84009520 con 0x7f7c94100cd0 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c94197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.477+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 0x7f7c9419f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f7c84009fd0 tx=0x7f7c8400b840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c8affd700 1 -- 172.21.15.40:0/2862346873 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c8401c070 con 0x7f7c94100cd0 2023-07-14T11:11:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c9419fb50 con 0x7f7c94100cd0 2023-07-14T11:11:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c8affd700 1 -- 172.21.15.40:0/2862346873 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c8400f7d0 con 0x7f7c94100cd0 2023-07-14T11:11:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c8affd700 1 -- 172.21.15.40:0/2862346873 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c84004b10 con 0x7f7c94100cd0 2023-07-14T11:11:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.478+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c941a0020 con 0x7f7c94100cd0 2023-07-14T11:11:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.483+0000 7f7c8affd700 1 -- 172.21.15.40:0/2862346873 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f7c84020070 con 0x7f7c94100cd0 2023-07-14T11:11:14.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 Processor -- start 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 -- start start 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 0x5566e0bf6690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf6c20 0x5566e0bf7090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0afac60 con 0x5566e0bf6c20 2023-07-14T11:11:14.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0b4d9f0 con 0x5566e0bf5480 2023-07-14T11:11:14.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0b46e80 con 0x5566e0bf5dd0 2023-07-14T11:11:14.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.484+0000 7f7c9a79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:32786/0 (socket says 172.21.15.40:32786) 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 -- 172.21.15.40:0/1273174213 learned_addr learned my addr 172.21.15.40:0/1273174213 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9979a700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf6c20 0x5566e0bf7090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 0x5566e0bf6690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c8affd700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 0x7f7c800a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 -- 172.21.15.40:0/1273174213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 msgr2=0x5566e0bf6690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 0x5566e0bf6690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 -- 172.21.15.40:0/1273174213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf6c20 msgr2=0x5566e0bf7090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf6c20 0x5566e0bf7090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 -- 172.21.15.40:0/1273174213 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5566e0974d90 con 0x5566e0bf5480 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c8affd700 1 -- 172.21.15.40:0/2862346873 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f7c840cd940 con 0x7f7c94100cd0 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 0x5566e0bf6690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 0x7f7c800a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.485+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c007530 tx=0x7f7c8c007840 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8ff79a0651663899 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7c7ffff700 1 -- 172.21.15.40:0/1273174213 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c8c00e040 con 0x5566e0bf5480 2023-07-14T11:11:14.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7c7ffff700 1 -- 172.21.15.40:0/1273174213 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c8c00f040 con 0x5566e0bf5480 2023-07-14T11:11:14.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7c7ffff700 1 -- 172.21.15.40:0/1273174213 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c8c017890 con 0x5566e0bf5480 2023-07-14T11:11:14.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/1273174213 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 msgr2=0x5566e0bf5890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 0x7f7c800a3600 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c020000 tx=0x7f7c8c021000 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.486+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c007530 tx=0x7f7c8c007840 comp rx=0 tx=0).stop 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/1273174213 shutdown_connections 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf6c20 0x5566e0bf7090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5480 0x5566e0bf5890 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/1273174213 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf5dd0 0x5566e0bf6690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/1273174213 >> 172.21.15.40:0/1273174213 conn(0x5566e0bead50 msgr2=0x5566e0bf0180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/1273174213 shutdown_connections 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/1273174213 wait complete. 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 -- start start 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.487+0000 7f7cafa5eb80 1 Processor -- start 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0bff120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c066a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0ae6a70 con 0x5566e0bf5480 2023-07-14T11:11:14.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0ae6770 con 0x5566e0bf5dd0 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e09ca8d0 con 0x5566e0bf6c20 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7cafa5eb80 10 client.0 _ll_register_callbacks cb 0x5566e0ae4710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0bff120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:32790/0 (socket says 172.21.15.40:32790) 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0bff120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42488/0 (socket says 172.21.15.40:42488) 2023-07-14T11:11:14.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 -- 172.21.15.40:0/2437470417 learned_addr learned my addr 172.21.15.40:0/2437470417 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c066a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 -- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 msgr2=0x5566e0c066a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c066a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 -- 172.21.15.40:0/2437470417 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 msgr2=0x5566e0bff120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.488+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0bff120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c9979a700 1 -- 172.21.15.40:0/2437470417 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c90009cd0 con 0x5566e0bf5dd0 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7c84096f30 tx=0x7f7c840d1040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=aa3cbfdc4bbfc567 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c7d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c84004e70 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5566e0aa04a0 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f7c8409aa30 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7c8409a570 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7cafa5eb80 10 client.25133 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f7c8c009140 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.489+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5566e0aa04a0 con 0x5566e0bf5dd0 2023-07-14T11:11:14.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f7c8409ac90 con 0x5566e0bf5dd0 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 10 client.25133 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 10 client.25133 resolve_mds: resolved name 'c' to daemon mds.c 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 20 client.25133 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c09f50 0x5566e0c095d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 4 client.25133 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5566e0b3fe60 con 0x5566e0c09f50 2023-07-14T11:11:14.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.490+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c09f50 0x5566e0c095d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.491+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c09f50 0x5566e0c095d0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=40e758492510b9d server_cookie=14fc8ea4ce0001e0 in_seq=0 out_seq=0 2023-07-14T11:11:14.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.491+0000 7f7c7d7fa700 10 client.25133 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2437470417 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5566e0b3fe60 con 0x5566e0c09f50 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7c7d7fa700 10 client.25133 handle_command_reply: tid=0 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 client.25133 shutdown 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 msgr2=0x5566e0bff660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f7c84096f30 tx=0x7f7c840d1040 comp rx=0 tx=0).stop 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 shutdown_connections 2023-07-14T11:11:14.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c09f50 0x5566e0c095d0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c066a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7c7d7fa700 0 client.25133 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0bff120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2437470417 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0bff660 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 >> 172.21.15.40:0/2437470417 conn(0x5566e0bead50 msgr2=0x5566e0bef860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 shutdown_connections 2023-07-14T11:11:14.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2437470417 wait complete. 2023-07-14T11:11:14.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.492+0000 7f7cafa5eb80 20 client.25133 trim_cache size 0 max 16384 2023-07-14T11:11:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.510+0000 7f7cafa5eb80 1 Processor -- start 2023-07-14T11:11:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 -- start start 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c06660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0bef860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0aa04a0 con 0x5566e0bf5480 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0af0470 con 0x5566e0bf5dd0 2023-07-14T11:11:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0b15790 con 0x5566e0bf6c20 2023-07-14T11:11:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42496/0 (socket says 172.21.15.40:42496) 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7c99f9b700 1 -- 172.21.15.40:0/4037777761 learned_addr learned my addr 172.21.15.40:0/4037777761 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0bef860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.511+0000 7f7c9979a700 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c06660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 -- 172.21.15.40:0/4037777761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 msgr2=0x5566e0bef860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0bef860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 -- 172.21.15.40:0/4037777761 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 msgr2=0x5566e0c06660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c06660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 -- 172.21.15.40:0/4037777761 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5566e0bec5d0 con 0x5566e0bf5480 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f7c90005fd0 tx=0x7f7c90009310 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9934401605e8310d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c88ff9700 1 -- 172.21.15.40:0/4037777761 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c90017040 con 0x5566e0bf5480 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c88ff9700 1 -- 172.21.15.40:0/4037777761 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c90018040 con 0x5566e0bf5480 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.512+0000 7f7c88ff9700 1 -- 172.21.15.40:0/4037777761 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c9001f540 con 0x5566e0bf5480 2023-07-14T11:11:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/4037777761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 msgr2=0x5566e0c095d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f7c90005fd0 tx=0x7f7c90009310 comp rx=0 tx=0).stop 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/4037777761 shutdown_connections 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0bef860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c095d0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/4037777761 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c06660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/4037777761 >> 172.21.15.40:0/4037777761 conn(0x5566e0bead50 msgr2=0x5566e0bfde00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/4037777761 shutdown_connections 2023-07-14T11:11:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/4037777761 wait complete. 2023-07-14T11:11:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.513+0000 7f7cafa5eb80 1 -- start start 2023-07-14T11:11:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 Processor -- start 2023-07-14T11:11:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c02da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c05760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0c05ca0 con 0x5566e0bf5480 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0c05e10 con 0x5566e0bf5dd0 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566e0c05f80 con 0x5566e0bf6c20 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7cafa5eb80 10 client.0 _ll_register_callbacks cb 0x5566e0ae4710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7c99f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42510/0 (socket says 172.21.15.40:42510) 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7c99f9b700 1 -- 172.21.15.40:0/2163594706 learned_addr learned my addr 172.21.15.40:0/2163594706 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7c9979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c02da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.514+0000 7f7c9a79c700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c05760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 -- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 msgr2=0x5566e0c05760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c05760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 -- 172.21.15.40:0/2163594706 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 msgr2=0x5566e0c02da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c02da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 -- 172.21.15.40:0/2163594706 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c900120f0 con 0x5566e0bf5480 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c02da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c99f9b700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f7c9001f1d0 tx=0x7f7c9001f360 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1b8c6d40527ff40e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c7d7fa700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7c90017070 con 0x5566e0bf5480 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f7c90012a30 con 0x5566e0bf5480 2023-07-14T11:11:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.515+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7c8c00f040 con 0x5566e0bf5480 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7c90012bf0 con 0x5566e0bf5480 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 10 client.15144 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5566e095dc10 con 0x5566e0bf5480 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7c8c00f040 con 0x5566e0bf5480 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f7c90010450 con 0x5566e0bf5480 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 10 client.15144 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 10 client.15144 resolve_mds: resolved name 'c' to daemon mds.c 2023-07-14T11:11:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 20 client.15144 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c63df0 0x5566e0c60740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 4 client.15144 mds_command: new command op to 24523 tid=0[{"prefix": "status"}] 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5566e0b02000 con 0x5566e0c63df0 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.516+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c63df0 0x5566e0c60740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7c9979a700 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c63df0 0x5566e0c60740 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=44883bd506009777 server_cookie=521624ae0f2cff18 in_seq=0 out_seq=0 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7c7d7fa700 10 client.15144 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7c7d7fa700 1 -- 172.21.15.40:0/2163594706 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+327 (crc 0 0 0) 0x5566e0b02000 con 0x5566e0c63df0 2023-07-14T11:11:14.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7c7d7fa700 10 client.15144 handle_command_reply: tid=0 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 client.15144 shutdown 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 msgr2=0x5566e0c02860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f7c9001f1d0 tx=0x7f7c9001f360 comp rx=0 tx=0).stop 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 shutdown_connections 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x5566e0c63df0 0x5566e0c60740 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566e0bf6c20 0x5566e0c05760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566e0bf5480 0x5566e0c02860 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7c7d7fa700 0 client.15144 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:14.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 --2- 172.21.15.40:0/2163594706 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566e0bf5dd0 0x5566e0c02da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.517+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 >> 172.21.15.40:0/2163594706 conn(0x5566e0bead50 msgr2=0x5566e0c095d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 shutdown_connections 2023-07-14T11:11:14.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7cafa5eb80 1 -- 172.21.15.40:0/2163594706 wait complete. 2023-07-14T11:11:14.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7cafa5eb80 20 client.15144 trim_cache size 0 max 16384 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 2, 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24523, 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 60.170179404000002, 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 109.448112571 2023-07-14T11:11:14.530 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:14.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 msgr2=0x7f7c800a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 0x7f7c800a3600 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7c8c020000 tx=0x7f7c8c021000 comp rx=0 tx=0).stop 2023-07-14T11:11:14.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 msgr2=0x7f7c94197fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f7c84009fd0 tx=0x7f7c8400b840 comp rx=0 tx=0).stop 2023-07-14T11:11:14.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 shutdown_connections 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f7c800a1150 0x7f7c800a3600 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7c94101660 0x7f7c9419f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7c940ffae0 0x7f7c94197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 --2- 172.21.15.40:0/2862346873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7c94100cd0 0x7f7c94197fc0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.518+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 >> 172.21.15.40:0/2862346873 conn(0x7f7c940fb0e0 msgr2=0x7f7c940fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.519+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 shutdown_connections 2023-07-14T11:11:14.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.519+0000 7f7c9c1ff700 1 -- 172.21.15.40:0/2862346873 wait complete. 2023-07-14T11:11:14.539 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.a status 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 -- 172.21.15.40:0/4065382388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a78100ce0 msgr2=0x7f1a78101130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/4065382388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a78100ce0 0x7f1a78101130 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f1a600098f0 tx=0x7f1a60009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 -- 172.21.15.40:0/4065382388 shutdown_connections 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/4065382388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78101670 0x7f1a78107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/4065382388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a780ffaf0 0x7f1a780fff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/4065382388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a78100ce0 0x7f1a78101130 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 -- 172.21.15.40:0/4065382388 >> 172.21.15.40:0/4065382388 conn(0x7f1a780fb130 msgr2=0x7f1a780fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 -- 172.21.15.40:0/4065382388 shutdown_connections 2023-07-14T11:11:14.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.742+0000 7f1a7cfff700 1 -- 172.21.15.40:0/4065382388 wait complete. 2023-07-14T11:11:14.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.743+0000 7f1a7cfff700 1 Processor -- start 2023-07-14T11:11:14.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.743+0000 7f1a7cfff700 1 -- start start 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.743+0000 7f1a7cfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.743+0000 7f1a7cfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78100ce0 0x7f1a78198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7cfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 0x7f1a7819f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7cfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a7819f670 con 0x7f1a78101670 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7cfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a7819f7b0 con 0x7f1a780ffaf0 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7cfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a7819f940 con 0x7f1a78100ce0 2023-07-14T11:11:14.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 0x7f1a7819f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:32830/0 (socket says 172.21.15.40:32830) 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7659c700 1 -- 172.21.15.40:0/184492485 learned_addr learned my addr 172.21.15.40:0/184492485 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78100ce0 0x7f1a78198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 -- 172.21.15.40:0/184492485 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78100ce0 msgr2=0x7f1a78198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78100ce0 0x7f1a78198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 -- 172.21.15.40:0/184492485 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 msgr2=0x7f1a78197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 -- 172.21.15.40:0/184492485 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a60009520 con 0x7f1a78101670 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a7659c700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.744+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 0x7f1a7819f0a0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f1a70007f10 tx=0x7f1a7000d3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.745+0000 7f1a6f7fe700 1 -- 172.21.15.40:0/184492485 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a70009ea0 con 0x7f1a78101670 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.745+0000 7f1a6f7fe700 1 -- 172.21.15.40:0/184492485 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1a7000f040 con 0x7f1a78101670 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.745+0000 7f1a6f7fe700 1 -- 172.21.15.40:0/184492485 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a70004350 con 0x7f1a78101670 2023-07-14T11:11:14.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.745+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a7819fc20 con 0x7f1a78101670 2023-07-14T11:11:14.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.745+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a781a0130 con 0x7f1a78101670 2023-07-14T11:11:14.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.748+0000 7f1a6f7fe700 1 -- 172.21.15.40:0/184492485 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1a7000db20 con 0x7f1a78101670 2023-07-14T11:11:14.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.749+0000 7f1a909f6b80 1 Processor -- start 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 -- start start 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a6f7fe700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 0x7f1a640a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace3c690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608ace3d090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acd40c60 con 0x5608ace3bdd0 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acd939f0 con 0x5608ace3cc20 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acd8ce80 con 0x5608ace3b480 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a6f7fe700 1 -- 172.21.15.40:0/184492485 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f1a700cc4a0 con 0x7f1a78101670 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 0x7f1a640a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50738/0 (socket says 172.21.15.40:50738) 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a7659c700 1 -- 172.21.15.40:0/1379120217 learned_addr learned my addr 172.21.15.40:0/1379120217 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608ace3d090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.750+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace3c690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 -- 172.21.15.40:0/1379120217 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 msgr2=0x5608ace3b890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 -- 172.21.15.40:0/1379120217 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 msgr2=0x5608ace3c690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace3c690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 -- 172.21.15.40:0/1379120217 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5608acbbad90 con 0x5608ace3cc20 2023-07-14T11:11:14.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a7659c700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608ace3d090 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1a70000f80 tx=0x7f1a70099350 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5427004d486f1bd2 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 0x7f1a640a3680 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f1a60005e90 tx=0x7f1a6000bf90 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a6cff9700 1 -- 172.21.15.40:0/1379120217 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a7009d530 con 0x5608ace3cc20 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a6cff9700 1 -- 172.21.15.40:0/1379120217 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1a700a1950 con 0x5608ace3cc20 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.751+0000 7f1a6cff9700 1 -- 172.21.15.40:0/1379120217 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a700840d0 con 0x5608ace3cc20 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 -- 172.21.15.40:0/1379120217 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 msgr2=0x5608ace3d090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608ace3d090 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f1a70000f80 tx=0x7f1a70099350 comp rx=0 tx=0).stop 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 -- 172.21.15.40:0/1379120217 shutdown_connections 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608ace3d090 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3b480 0x5608ace3b890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/1379120217 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace3c690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 -- 172.21.15.40:0/1379120217 >> 172.21.15.40:0/1379120217 conn(0x5608ace30d50 msgr2=0x5608ace36180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 -- 172.21.15.40:0/1379120217 shutdown_connections 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.752+0000 7f1a909f6b80 1 -- 172.21.15.40:0/1379120217 wait complete. 2023-07-14T11:11:14.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 -- start start 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 Processor -- start 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace45120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace4c6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.753+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acd2ca70 con 0x5608ace3bdd0 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acd2c770 con 0x5608ace3b480 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acc108d0 con 0x5608ace3cc20 2023-07-14T11:11:14.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a909f6b80 10 client.0 _ll_register_callbacks cb 0x5608acd2a710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42532/0 (socket says 172.21.15.40:42532) 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 -- 172.21.15.40:0/860622323 learned_addr learned my addr 172.21.15.40:0/860622323 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a7659c700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace45120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace4c6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 -- 172.21.15.40:0/860622323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 msgr2=0x5608ace4c6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace4c6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 -- 172.21.15.40:0/860622323 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 msgr2=0x5608ace45120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace45120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 -- 172.21.15.40:0/860622323 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a68009e40 con 0x5608ace3bdd0 2023-07-14T11:11:14.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f1a60016000 tx=0x7f1a60005dc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=752849db7b726c5e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.754+0000 7f1a5dffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a60010e30 con 0x5608ace3bdd0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5608acce64a0 con 0x5608ace3bdd0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f1a6000b710 con 0x5608ace3bdd0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1a6000b8d0 con 0x5608ace3bdd0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a909f6b80 10 client.15156 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f1a781a0130 con 0x5608ace3bdd0 2023-07-14T11:11:14.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5608acce64a0 con 0x5608ace3bdd0 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.755+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1a60028470 con 0x5608ace3bdd0 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 10 client.15156 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 10 client.15156 resolve_mds: resolved name 'a' to daemon mds.a 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 20 client.15156 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608ace503c0 0x5608ace4f7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 4 client.15156 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5608acd85e60 con 0x5608ace503c0 2023-07-14T11:11:14.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a7659c700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608ace503c0 0x5608ace4f7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a7659c700 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608ace503c0 0x5608ace4f7f0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=4d69adbf74768ddc server_cookie=b0a37cb738167446 in_seq=0 out_seq=0 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.756+0000 7f1a5dffb700 10 client.15156 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.757+0000 7f1a5dffb700 1 -- 172.21.15.40:0/860622323 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5608acd85e60 con 0x5608ace503c0 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.757+0000 7f1a5dffb700 10 client.15156 handle_command_reply: tid=0 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.757+0000 7f1a909f6b80 1 client.15156 shutdown 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 msgr2=0x5608ace45660 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f1a60016000 tx=0x7f1a60005dc0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 shutdown_connections 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608ace503c0 0x5608ace4f7f0 crc :-1 s=READY pgs=19 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace4c6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace45120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a5dffb700 0 client.15156 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/860622323 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace45660 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 >> 172.21.15.40:0/860622323 conn(0x5608ace30d50 msgr2=0x5608ace35860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 shutdown_connections 2023-07-14T11:11:14.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 1 -- 172.21.15.40:0/860622323 wait complete. 2023-07-14T11:11:14.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.758+0000 7f1a909f6b80 20 client.15156 trim_cache size 0 max 16384 2023-07-14T11:11:14.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.776+0000 7f1a909f6b80 1 Processor -- start 2023-07-14T11:11:14.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.776+0000 7f1a909f6b80 1 -- start start 2023-07-14T11:11:14.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace35860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace43030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acce64a0 con 0x5608ace3bdd0 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608ace3c520 con 0x5608ace3b480 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608ace46310 con 0x5608ace3cc20 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42536/0 (socket says 172.21.15.40:42536) 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 -- 172.21.15.40:0/2755973482 learned_addr learned my addr 172.21.15.40:0/2755973482 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a7659c700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace35860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace43030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 -- 172.21.15.40:0/2755973482 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 msgr2=0x5608ace43030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace43030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 -- 172.21.15.40:0/2755973482 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 msgr2=0x5608ace35860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace35860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a6ffff700 1 -- 172.21.15.40:0/2755973482 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5608acba3c10 con 0x5608ace3bdd0 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.777+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace43030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:11:14.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.778+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f1a60005a80 tx=0x7f1a6000b2e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b33bd7b66eb1834b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.778+0000 7f1a6d7fa700 1 -- 172.21.15.40:0/2755973482 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a6000f7a0 con 0x5608ace3bdd0 2023-07-14T11:11:14.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.778+0000 7f1a6d7fa700 1 -- 172.21.15.40:0/2755973482 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1a6000f460 con 0x5608ace3bdd0 2023-07-14T11:11:14.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.778+0000 7f1a6d7fa700 1 -- 172.21.15.40:0/2755973482 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a6000fbb0 con 0x5608ace3bdd0 2023-07-14T11:11:14.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- 172.21.15.40:0/2755973482 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 msgr2=0x5608ace4c630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f1a60005a80 tx=0x7f1a6000b2e0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- 172.21.15.40:0/2755973482 shutdown_connections 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3cc20 0x5608ace43030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3b480 0x5608ace35860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/2755973482 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3bdd0 0x5608ace4c630 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- 172.21.15.40:0/2755973482 >> 172.21.15.40:0/2755973482 conn(0x5608ace30d50 msgr2=0x5608ace43e00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- 172.21.15.40:0/2755973482 shutdown_connections 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- 172.21.15.40:0/2755973482 wait complete. 2023-07-14T11:11:14.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 -- start start 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.779+0000 7f1a909f6b80 1 Processor -- start 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 0x5608ace4c5c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608acea4240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acea4780 con 0x5608ace3b480 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acea48c0 con 0x5608ace3cc20 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5608acea4a30 con 0x5608ace3bdd0 2023-07-14T11:11:14.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a909f6b80 10 client.0 _ll_register_callbacks cb 0x7f1a60005b40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42546/0 (socket says 172.21.15.40:42546) 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 -- 172.21.15.40:0/3351710109 learned_addr learned my addr 172.21.15.40:0/3351710109 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a76d9d700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608acea4240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 0x5608ace4c5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 -- 172.21.15.40:0/3351710109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 msgr2=0x5608ace4c5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 0x5608ace4c5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 -- 172.21.15.40:0/3351710109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 msgr2=0x5608acea4240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.780+0000 7f1a7659c700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608acea4240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a7659c700 1 -- 172.21.15.40:0/3351710109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a781a0130 con 0x5608ace3b480 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 0x5608ace4c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a7659c700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f1a68012860 tx=0x7f1a6801fd20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c2a1eeb01ff66702 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a5dffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1a68017040 con 0x5608ace3b480 2023-07-14T11:11:14.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1a700a1950 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f1a680123d0 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1a6801f600 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a909f6b80 10 client.15162 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.781+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5608acbe1200 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1a700a1950 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1a680275e0 con 0x5608ace3b480 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 10 client.15162 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:14.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 10 client.15162 resolve_mds: resolved name 'a' to daemon mds.a 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 20 client.15162 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608acea9760 0x5608acea6090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 4 client.15162 mds_command: new command op to 24565 tid=0[{"prefix": "status"}] 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5608acd48000 con 0x5608acea9760 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.782+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608acea9760 0x5608acea6090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a6ffff700 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608acea9760 0x5608acea6090 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=906abe755e538e25 server_cookie=a187744e81ae7d12 in_seq=0 out_seq=0 2023-07-14T11:11:14.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a5dffb700 10 client.15162 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:14.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a5dffb700 1 -- 172.21.15.40:0/3351710109 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x5608acd48000 con 0x5608acea9760 2023-07-14T11:11:14.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a5dffb700 10 client.15162 handle_command_reply: tid=0 2023-07-14T11:11:14.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 client.15162 shutdown 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 msgr2=0x5608ace42300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f1a68012860 tx=0x7f1a6801fd20 comp rx=0 tx=0).stop 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 shutdown_connections 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5608acea9760 0x5608acea6090 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5608ace3cc20 0x5608acea4240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5608ace3b480 0x5608ace42300 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a5dffb700 0 client.15162 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 --2- 172.21.15.40:0/3351710109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5608ace3bdd0 0x5608ace4c5c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 >> 172.21.15.40:0/3351710109 conn(0x5608ace30d50 msgr2=0x5608ace35860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 shutdown_connections 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.783+0000 7f1a909f6b80 1 -- 172.21.15.40:0/3351710109 wait complete. 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a909f6b80 20 client.15162 trim_cache size 0 max 16384 2023-07-14T11:11:14.794 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 1, 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24565, 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 61.011403512000001, 2023-07-14T11:11:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 98.946055506999997 2023-07-14T11:11:14.796 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:14.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 msgr2=0x7f1a640a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 0x7f1a640a3680 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f1a60005e90 tx=0x7f1a6000bf90 comp rx=0 tx=0).stop 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 msgr2=0x7f1a7819f0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 0x7f1a7819f0a0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f1a70007f10 tx=0x7f1a7000d3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 shutdown_connections 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1a640a11d0 0x7f1a640a3680 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a78101670 0x7f1a7819f0a0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1a780ffaf0 0x7f1a78197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 --2- 172.21.15.40:0/184492485 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1a78100ce0 0x7f1a78198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:14.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 >> 172.21.15.40:0/184492485 conn(0x7f1a780fb130 msgr2=0x7f1a780fd380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:14.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 shutdown_connections 2023-07-14T11:11:14.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:14.784+0000 7f1a7cfff700 1 -- 172.21.15.40:0/184492485 wait complete. 2023-07-14T11:11:14.805 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2023-07-14T11:11:15.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.005+0000 7f6a651ff700 1 -- 172.21.15.40:0/3545431410 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60100cd0 msgr2=0x7f6a60101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.005+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3545431410 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60100cd0 0x7f6a60101120 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f6a54009990 tx=0x7f6a54009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 -- 172.21.15.40:0/3545431410 shutdown_connections 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3545431410 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60101660 0x7f6a60107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3545431410 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3545431410 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60100cd0 0x7f6a60101120 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 -- 172.21.15.40:0/3545431410 >> 172.21.15.40:0/3545431410 conn(0x7f6a600fb0e0 msgr2=0x7f6a600fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 -- 172.21.15.40:0/3545431410 shutdown_connections 2023-07-14T11:11:15.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.006+0000 7f6a651ff700 1 -- 172.21.15.40:0/3545431410 wait complete. 2023-07-14T11:11:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 Processor -- start 2023-07-14T11:11:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 -- start start 2023-07-14T11:11:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60100cd0 0x7f6a60072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a600735e0 con 0x7f6a60101660 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a60073750 con 0x7f6a60100cd0 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.007+0000 7f6a651ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6a6019f870 con 0x7f6a600ffae0 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5ed9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60100cd0 0x7f6a60072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42562/0 (socket says 172.21.15.40:42562) 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 -- 172.21.15.40:0/3450200889 learned_addr learned my addr 172.21.15.40:0/3450200889 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5ed9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50794/0 (socket says 172.21.15.40:50794) 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 -- 172.21.15.40:0/3450200889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 msgr2=0x7f6a600742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 -- 172.21.15.40:0/3450200889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60100cd0 msgr2=0x7f6a60072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60100cd0 0x7f6a60072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 -- 172.21.15.40:0/3450200889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6a540095f0 con 0x7f6a60101660 2023-07-14T11:11:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f6a5800ef30 tx=0x7f6a5800c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.008+0000 7f6a47fff700 1 -- 172.21.15.40:0/3450200889 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a58018070 con 0x7f6a60101660 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.009+0000 7f6a47fff700 1 -- 172.21.15.40:0/3450200889 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6a5800f040 con 0x7f6a60101660 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.009+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6a6019faf0 con 0x7f6a60101660 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.009+0000 7f6a47fff700 1 -- 172.21.15.40:0/3450200889 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a5800cd00 con 0x7f6a60101660 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.009+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a601a0050 con 0x7f6a60101660 2023-07-14T11:11:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.012+0000 7f6a47fff700 1 -- 172.21.15.40:0/3450200889 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6a58003a00 con 0x7f6a60101660 2023-07-14T11:11:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.013+0000 7f6a78b17b80 1 Processor -- start 2023-07-14T11:11:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.013+0000 7f6a78b17b80 1 -- start start 2023-07-14T11:11:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a47fff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 0x7f6a480a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a47fff700 1 -- 172.21.15.40:0/3450200889 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f6a580cc720 con 0x7f6a60101660 2023-07-14T11:11:15.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d0e690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d0f090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36c12c60 con 0x557d36d0ddd0 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36c659f0 con 0x557d36d0d480 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36c5ee80 con 0x557d36d0ec20 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 0x7f6a480a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5ed9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5ed9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:32882/0 (socket says 172.21.15.40:32882) 2023-07-14T11:11:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5ed9d700 1 -- 172.21.15.40:0/2981288153 learned_addr learned my addr 172.21.15.40:0/2981288153 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d0f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.014+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d0e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 -- 172.21.15.40:0/2981288153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 msgr2=0x557d36d0f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d0f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 -- 172.21.15.40:0/2981288153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 msgr2=0x557d36d0e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d0e690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 -- 172.21.15.40:0/2981288153 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d36a8cd90 con 0x557d36d0d480 2023-07-14T11:11:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f6a50009d30 tx=0x7f6a50011720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6b5687464e3da294 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 0x7f6a480a3680 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f6a5400b620 tx=0x7f6a540054f0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a457fa700 1 -- 172.21.15.40:0/2981288153 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a50004220 con 0x557d36d0d480 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a457fa700 1 -- 172.21.15.40:0/2981288153 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6a50004380 con 0x557d36d0d480 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.015+0000 7f6a457fa700 1 -- 172.21.15.40:0/2981288153 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a50019810 con 0x557d36d0d480 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2981288153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 msgr2=0x557d36d0d890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f6a50009d30 tx=0x7f6a50011720 comp rx=0 tx=0).stop 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2981288153 shutdown_connections 2023-07-14T11:11:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d0f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d0d890 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2981288153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d0e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2981288153 >> 172.21.15.40:0/2981288153 conn(0x557d36d02d50 msgr2=0x557d36d08180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2981288153 shutdown_connections 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.016+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2981288153 wait complete. 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.017+0000 7f6a78b17b80 1 -- start start 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.017+0000 7f6a78b17b80 1 Processor -- start 2023-07-14T11:11:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 0x557d36d170a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0ec20 0x557d36d1e620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36bfea70 con 0x557d36d0ddd0 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36bfe770 con 0x557d36d0ec20 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36ae28d0 con 0x557d36d0d480 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a78b17b80 10 client.0 _ll_register_callbacks cb 0x557d36bfc710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42572/0 (socket says 172.21.15.40:42572) 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a5e59c700 1 -- 172.21.15.40:0/674557602 learned_addr learned my addr 172.21.15.40:0/674557602 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 0x557d36d170a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.018+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0ec20 0x557d36d1e620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 -- 172.21.15.40:0/674557602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 msgr2=0x557d36d170a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 0x557d36d170a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 -- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0ec20 msgr2=0x557d36d1e620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0ec20 0x557d36d1e620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 -- 172.21.15.40:0/674557602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6a50009960 con 0x557d36d0ddd0 2023-07-14T11:11:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 0x557d36d170a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f6a54009990 tx=0x7f6a54009ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=41ae3955a3b49d2f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a42ff5700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a5401d040 con 0x557d36d0ddd0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557d36bb84a0 con 0x557d36d0ddd0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f6a54004af0 con 0x557d36d0ddd0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6a54004cb0 con 0x557d36d0ddd0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a78b17b80 10 client.15174 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.019+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x557d36a75c10 con 0x557d36d0ddd0 2023-07-14T11:11:15.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557d36bb84a0 con 0x557d36d0ddd0 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f6a54032440 con 0x557d36d0ddd0 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 10 client.15174 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 10 client.15174 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 20 client.15174 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d223b0 0x557d36d21770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 4 client.15174 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:15.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557d36c57e60 con 0x557d36d223b0 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.020+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d223b0 0x557d36d21770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.021+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d223b0 0x557d36d21770 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f2547a7bc71ca68d server_cookie=daff0e1bd1f5d938 in_seq=0 out_seq=0 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.021+0000 7f6a42ff5700 10 client.15174 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a42ff5700 1 -- 172.21.15.40:0/674557602 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x557d36c57e60 con 0x557d36d223b0 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a42ff5700 10 client.15174 handle_command_reply: tid=0 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 client.15174 shutdown 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 msgr2=0x557d36d175e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f6a54009990 tx=0x7f6a54009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 shutdown_connections 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d223b0 0x557d36d21770 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0ec20 0x557d36d1e620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0d480 0x557d36d170a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/674557602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d175e0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a42ff5700 0 client.15174 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 >> 172.21.15.40:0/674557602 conn(0x557d36d02d50 msgr2=0x557d36d07860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 shutdown_connections 2023-07-14T11:11:15.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 1 -- 172.21.15.40:0/674557602 wait complete. 2023-07-14T11:11:15.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.022+0000 7f6a78b17b80 20 client.15174 trim_cache size 0 max 16384 2023-07-14T11:11:15.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 Processor -- start 2023-07-14T11:11:15.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 -- start start 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d15030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d21770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36bb84a0 con 0x557d36d0ddd0 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36d20310 con 0x557d36d0d480 2023-07-14T11:11:15.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.041+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36d18290 con 0x557d36d0ec20 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42588/0 (socket says 172.21.15.40:42588) 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 -- 172.21.15.40:0/58804726 learned_addr learned my addr 172.21.15.40:0/58804726 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d15030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d21770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 -- 172.21.15.40:0/58804726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 msgr2=0x557d36d21770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d21770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 -- 172.21.15.40:0/58804726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 msgr2=0x557d36d15030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d15030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 -- 172.21.15.40:0/58804726 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557d36ab3200 con 0x557d36d0ddd0 2023-07-14T11:11:15.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d15030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:15.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.042+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f6a540098e0 tx=0x7f6a5400b2b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e03ec008ee6b941c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.049 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a45ffb700 1 -- 172.21.15.40:0/58804726 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a5401d040 con 0x557d36d0ddd0 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a45ffb700 1 -- 172.21.15.40:0/58804726 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6a54009290 con 0x557d36d0ddd0 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a45ffb700 1 -- 172.21.15.40:0/58804726 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a5400f460 con 0x557d36d0ddd0 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 -- 172.21.15.40:0/58804726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 msgr2=0x557d36d1e5e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f6a540098e0 tx=0x7f6a5400b2b0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 -- 172.21.15.40:0/58804726 shutdown_connections 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d21770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d15030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/58804726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d1e5e0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 -- 172.21.15.40:0/58804726 >> 172.21.15.40:0/58804726 conn(0x557d36d02d50 msgr2=0x557d36d2b140 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 -- 172.21.15.40:0/58804726 shutdown_connections 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.043+0000 7f6a78b17b80 1 -- 172.21.15.40:0/58804726 wait complete. 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 -- start start 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 Processor -- start 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d13790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.051 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d76270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36d1e780 con 0x557d36d0ddd0 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36d1e8f0 con 0x557d36d0d480 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557d36d767b0 con 0x557d36d0ec20 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.044+0000 7f6a78b17b80 10 client.0 _ll_register_callbacks cb 0x7f6a5400ba20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42598/0 (socket says 172.21.15.40:42598) 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 -- 172.21.15.40:0/2438194412 learned_addr learned my addr 172.21.15.40:0/2438194412 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d13790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d76270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 -- 172.21.15.40:0/2438194412 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 msgr2=0x557d36d76270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d76270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 -- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 msgr2=0x557d36d13790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d13790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.053 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 -- 172.21.15.40:0/2438194412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557d36a8cd90 con 0x557d36d0ddd0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5f59e700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d76270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a5e59c700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f6a5400c010 tx=0x7f6a54005d00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d9ffb326dc9a2e60 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.045+0000 7f6a42ff5700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6a5401d030 con 0x557d36d0ddd0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f6a50004380 con 0x557d36d0ddd0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f6a54004d80 con 0x557d36d0ddd0 2023-07-14T11:11:15.054 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f6a5400f460 con 0x557d36d0ddd0 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 10 client.15180 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x557d36a75c10 con 0x557d36d0ddd0 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6a50004380 con 0x557d36d0ddd0 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f6a54048060 con 0x557d36d0ddd0 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 10 client.15180 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 10 client.15180 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.046+0000 7f6a78b17b80 20 client.15180 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:15.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d7afb0 0x557d36d77960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:15.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a78b17b80 4 client.15180 mds_command: new command op to 24569 tid=0[{"prefix": "status"}] 2023-07-14T11:11:15.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "status"}) v1 -- 0x557d36c1a000 con 0x557d36d7afb0 2023-07-14T11:11:15.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d7afb0 0x557d36d77960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:15.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a5ed9d700 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d7afb0 0x557d36d77960 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9d237e8a1605e044 server_cookie=dec1b0ec81899ed8 in_seq=0 out_seq=0 2023-07-14T11:11:15.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.047+0000 7f6a42ff5700 10 client.15180 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:15.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a42ff5700 1 -- 172.21.15.40:0/2438194412 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+326 (crc 0 0 0) 0x557d36c1a000 con 0x557d36d7afb0 2023-07-14T11:11:15.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a42ff5700 10 client.15180 handle_command_reply: tid=0 2023-07-14T11:11:15.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 client.15180 shutdown 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 msgr2=0x557d36d13cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f6a5400c010 tx=0x7f6a54005d00 comp rx=0 tx=0).stop 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 shutdown_connections 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557d36d7afb0 0x557d36d77960 crc :-1 s=READY pgs=27 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557d36d0ec20 0x557d36d76270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557d36d0d480 0x557d36d13790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.058 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 --2- 172.21.15.40:0/2438194412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557d36d0ddd0 0x557d36d13cd0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a42ff5700 0 client.15180 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 >> 172.21.15.40:0/2438194412 conn(0x557d36d02d50 msgr2=0x557d36d15030 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 shutdown_connections 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 1 -- 172.21.15.40:0/2438194412 wait complete. 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.048+0000 7f6a78b17b80 20 client.15180 trim_cache size 0 max 16384 2023-07-14T11:11:15.059 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 0, 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24569, 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 91.432654083000003, 2023-07-14T11:11:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 96.969359716 2023-07-14T11:11:15.061 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:15.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 msgr2=0x7f6a480a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 0x7f6a480a3680 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f6a5400b620 tx=0x7f6a540054f0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 msgr2=0x7f6a60072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f6a5800ef30 tx=0x7f6a5800c3a0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 shutdown_connections 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6a480a11d0 0x7f6a480a3680 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6a60101660 0x7f6a60072e40 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6a600ffae0 0x7f6a600742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 --2- 172.21.15.40:0/3450200889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6a60100cd0 0x7f6a60072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:15.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 >> 172.21.15.40:0/3450200889 conn(0x7f6a600fb0e0 msgr2=0x7f6a600fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:15.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 shutdown_connections 2023-07-14T11:11:15.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:15.049+0000 7f6a651ff700 1 -- 172.21.15.40:0/3450200889 wait complete. 2023-07-14T11:11:15.069 INFO:tasks.cephadm:Setup complete, yielding 2023-07-14T11:11:15.070 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-07-14T11:11:15.079 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:11:15.080 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph orch status' 2023-07-14T11:11:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:15 smithi130 ceph-mon[129405]: pgmap v326: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 11 op/s 2023-07-14T11:11:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:15 smithi040 ceph-mon[97954]: pgmap v326: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 11 op/s 2023-07-14T11:11:15.812 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:15.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:15 smithi045 ceph-mon[128791]: pgmap v326: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 11 op/s 2023-07-14T11:11:17.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 -- 172.21.15.40:0/1113156479 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814072f70 msgr2=0x7f68140733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:17.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 --2- 172.21.15.40:0/1113156479 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814072f70 0x7f68140733e0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f68000099c0 tx=0x7f6800009cd0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 -- 172.21.15.40:0/1113156479 shutdown_connections 2023-07-14T11:11:17.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 --2- 172.21.15.40:0/1113156479 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f681410c170 0x7f681410e550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 --2- 172.21.15.40:0/1113156479 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6814074540 0x7f68140729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 --2- 172.21.15.40:0/1113156479 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814072f70 0x7f68140733e0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.318+0000 7f681ab68700 1 -- 172.21.15.40:0/1113156479 >> 172.21.15.40:0/1113156479 conn(0x7f68140fb1a0 msgr2=0x7f68140fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.319+0000 7f681ab68700 1 -- 172.21.15.40:0/1113156479 shutdown_connections 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.319+0000 7f681ab68700 1 -- 172.21.15.40:0/1113156479 wait complete. 2023-07-14T11:11:17.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 Processor -- start 2023-07-14T11:11:17.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 -- start start 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6814072f70 0x7f681419bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f681410c170 0x7f68141a34a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68141a3a70 con 0x7f6814074540 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68141a3bb0 con 0x7f681410c170 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f681ab68700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f68141a3d40 con 0x7f6814072f70 2023-07-14T11:11:17.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f6813fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f6813fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42618/0 (socket says 172.21.15.40:42618) 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f6813fff700 1 -- 172.21.15.40:0/3605724873 learned_addr learned my addr 172.21.15.40:0/3605724873 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f6818904700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6814072f70 0x7f681419bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.320+0000 7f6819105700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f681410c170 0x7f68141a34a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 -- 172.21.15.40:0/3605724873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6814072f70 msgr2=0x7f681419bee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6814072f70 0x7f681419bee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 -- 172.21.15.40:0/3605724873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f681410c170 msgr2=0x7f68141a34a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:17.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f681410c170 0x7f68141a34a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 -- 172.21.15.40:0/3605724873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68000095f0 con 0x7f6814074540 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6813fff700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f6800009990 tx=0x7f6800005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f680001d070 con 0x7f6814074540 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6800003f90 con 0x7f6814074540 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6800022d50 con 0x7f6814074540 2023-07-14T11:11:17.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.321+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f68141a3fc0 con 0x7f6814074540 2023-07-14T11:11:17.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.322+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f68141a4320 con 0x7f6814074540 2023-07-14T11:11:17.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.323+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6814101cc0 con 0x7f6814074540 2023-07-14T11:11:17.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.324+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6800022470 con 0x7f6814074540 2023-07-14T11:11:17.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.326+0000 7f6811ffb700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 0x7f68040a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:17.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.326+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f68000cdfe0 con 0x7f6814074540 2023-07-14T11:11:17.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.326+0000 7f6818904700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 0x7f68040a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:17.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.327+0000 7f6818904700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 0x7f68040a3680 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f6808007c60 tx=0x7f68080073d0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:17.332 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.330+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6800097550 con 0x7f6814074540 2023-07-14T11:11:17.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.553+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) v1 -- 0x7f6814060950 con 0x7f68040a11d0 2023-07-14T11:11:17.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.554+0000 7f6811ffb700 1 -- 172.21.15.40:0/3605724873 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+43 (secure 0 0 0) 0x7f6814060950 con 0x7f68040a11d0 2023-07-14T11:11:17.557 INFO:teuthology.orchestra.run.smithi040.stdout:Backend: cephadm 2023-07-14T11:11:17.557 INFO:teuthology.orchestra.run.smithi040.stdout:Available: Yes 2023-07-14T11:11:17.557 INFO:teuthology.orchestra.run.smithi040.stdout:Paused: No 2023-07-14T11:11:17.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 msgr2=0x7f68040a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 0x7f68040a3680 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f6808007c60 tx=0x7f68080073d0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 msgr2=0x7f681419c420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f6800009990 tx=0x7f6800005740 comp rx=0 tx=0).stop 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 shutdown_connections 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f68040a11d0 0x7f68040a3680 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f681410c170 0x7f68141a34a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6814072f70 0x7f681419bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 --2- 172.21.15.40:0/3605724873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6814074540 0x7f681419c420 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:17.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 >> 172.21.15.40:0/3605724873 conn(0x7f68140fb1a0 msgr2=0x7f68140fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:17.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 shutdown_connections 2023-07-14T11:11:17.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:17.557+0000 7f681ab68700 1 -- 172.21.15.40:0/3605724873 wait complete. 2023-07-14T11:11:17.999 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:17 smithi040 ceph-mon[97954]: pgmap v327: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:18.015 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph orch ps' 2023-07-14T11:11:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:17 smithi130 ceph-mon[129405]: pgmap v327: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:17 smithi045 ceph-mon[128791]: pgmap v327: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:18.752 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:19.041 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:18 smithi040 ceph-mon[97954]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:19.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:18 smithi130 ceph-mon[129405]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:19.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:18 smithi045 ceph-mon[128791]: from='client.15186 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:19 smithi130 ceph-mon[129405]: pgmap v328: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:19 smithi040 ceph-mon[97954]: pgmap v328: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:19 smithi045 ceph-mon[128791]: pgmap v328: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:11:20.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.407+0000 7f47642ba700 1 -- 172.21.15.40:0/3816541785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c101720 msgr2=0x7f475c107fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:20.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.407+0000 7f47642ba700 1 --2- 172.21.15.40:0/3816541785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c101720 0x7f475c107fe0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f475400b0a0 tx=0x7f475400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 -- 172.21.15.40:0/3816541785 shutdown_connections 2023-07-14T11:11:20.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 --2- 172.21.15.40:0/3816541785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c101720 0x7f475c107fe0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 --2- 172.21.15.40:0/3816541785 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c0ffb90 0x7f475c0fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 --2- 172.21.15.40:0/3816541785 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c100d90 0x7f475c1011e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 -- 172.21.15.40:0/3816541785 >> 172.21.15.40:0/3816541785 conn(0x7f475c0fb1a0 msgr2=0x7f475c0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:20.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 -- 172.21.15.40:0/3816541785 shutdown_connections 2023-07-14T11:11:20.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.408+0000 7f47642ba700 1 -- 172.21.15.40:0/3816541785 wait complete. 2023-07-14T11:11:20.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.409+0000 7f47642ba700 1 Processor -- start 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.409+0000 7f47642ba700 1 -- start start 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 0x7f475c195f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c101720 0x7f475c19cf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475c19d550 con 0x7f475c0ffb90 2023-07-14T11:11:20.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475c19d690 con 0x7f475c100d90 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f47642ba700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f475c19d820 con 0x7f475c101720 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f4762056700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f4762056700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42636/0 (socket says 172.21.15.40:42636) 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f4762056700 1 -- 172.21.15.40:0/456474516 learned_addr learned my addr 172.21.15.40:0/456474516 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f4761855700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 0x7f475c195f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.410+0000 7f4762857700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c101720 0x7f475c19cf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:20.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 -- 172.21.15.40:0/456474516 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c101720 msgr2=0x7f475c19cf80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c101720 0x7f475c19cf80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 -- 172.21.15.40:0/456474516 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 msgr2=0x7f475c195f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 0x7f475c195f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 -- 172.21.15.40:0/456474516 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4754009d00 con 0x7f475c0ffb90 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4761855700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 0x7f475c195f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:20.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f4762056700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f475800e9a0 tx=0x7f475800ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:20.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4758009790 con 0x7f475c0ffb90 2023-07-14T11:11:20.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f475c19db00 con 0x7f475c0ffb90 2023-07-14T11:11:20.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f475800c810 con 0x7f475c0ffb90 2023-07-14T11:11:20.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4758003c70 con 0x7f475c0ffb90 2023-07-14T11:11:20.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.411+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f475c19e000 con 0x7f475c0ffb90 2023-07-14T11:11:20.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.412+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f475c107110 con 0x7f475c0ffb90 2023-07-14T11:11:20.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.415+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f475800cc80 con 0x7f475c0ffb90 2023-07-14T11:11:20.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.417+0000 7f47537fe700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 0x7f47480a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:20.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.417+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4758014070 con 0x7f475c0ffb90 2023-07-14T11:11:20.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.417+0000 7f4761855700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 0x7f47480a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:20.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.418+0000 7f4761855700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 0x7f47480a3600 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f474c005950 tx=0x7f474c00b410 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:20.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.421+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f47580961e0 con 0x7f475c0ffb90 2023-07-14T11:11:20.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.640+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7f475c060950 con 0x7f47480a1150 2023-07-14T11:11:20.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.654+0000 7f47537fe700 1 -- 172.21.15.40:0/456474516 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+4185 (secure 0 0 0) 0x7f475c060950 con 0x7f47480a1150 2023-07-14T11:11:20.657 INFO:teuthology.orchestra.run.smithi040.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-07-14T11:11:20.657 INFO:teuthology.orchestra.run.smithi040.stdout:mds.a smithi040 running (105s) 21s ago 104s 17.0M - 18.0.0-4757-g3fc0089c afd12e232ec0 29e47f58b83b 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.b smithi045 running (102s) 31s ago 102s 22.9M - 18.0.0-4757-g3fc0089c afd12e232ec0 7ed87b201a08 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.c smithi130 running (115s) 31s ago 115s 20.9M - 18.0.0-4757-g3fc0089c afd12e232ec0 32e5999b480e 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.d smithi040 running (2m) 21s ago 2m 15.7M - 18.0.0-4757-g3fc0089c afd12e232ec0 3db246c6c8ae 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.e smithi045 running (2m) 31s ago 119s 18.2M - 18.0.0-4757-g3fc0089c afd12e232ec0 0d862ba0764b 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.f smithi130 running (118s) 31s ago 117s 20.7M - 18.0.0-4757-g3fc0089c afd12e232ec0 748a9c09cc2b 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.g smithi040 running (108s) 21s ago 107s 16.2M - 18.0.0-4757-g3fc0089c afd12e232ec0 2260d8b4dc34 2023-07-14T11:11:20.658 INFO:teuthology.orchestra.run.smithi040.stdout:mds.h smithi045 running (113s) 31s ago 113s 20.3M - 18.0.0-4757-g3fc0089c afd12e232ec0 e5845f21b107 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mds.i smithi130 running (111s) 31s ago 110s 19.4M - 18.0.0-4757-g3fc0089c afd12e232ec0 83a48a6123ce 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mds.j smithi040 running (2m) 21s ago 2m 13.5M - 18.0.0-4757-g3fc0089c afd12e232ec0 85248e8472bf 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mds.k smithi045 running (2m) 31s ago 2m 13.8M - 18.0.0-4757-g3fc0089c afd12e232ec0 9b13c4cc289f 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mds.l smithi130 running (2m) 31s ago 2m 18.3M - 18.0.0-4757-g3fc0089c afd12e232ec0 e9db87968f86 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mgr.x smithi040 *:9283,8765 running (9m) 21s ago 9m 520M - 18.0.0-4757-g3fc0089c afd12e232ec0 3b69bcb589de 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mgr.y smithi045 *:8443,8765 running (7m) 31s ago 7m 446M - 18.0.0-4757-g3fc0089c afd12e232ec0 07fcc2041cd9 2023-07-14T11:11:20.659 INFO:teuthology.orchestra.run.smithi040.stdout:mgr.z smithi130 *:8443,8765 running (7m) 31s ago 7m 446M - 18.0.0-4757-g3fc0089c afd12e232ec0 c7b2f42792ae 2023-07-14T11:11:20.660 INFO:teuthology.orchestra.run.smithi040.stdout:mon.a smithi040 running (9m) 21s ago 9m 92.5M 2048M 18.0.0-4757-g3fc0089c afd12e232ec0 65654455f8c8 2023-07-14T11:11:20.660 INFO:teuthology.orchestra.run.smithi040.stdout:mon.b smithi045 running (7m) 31s ago 7m 62.0M 2048M 18.0.0-4757-g3fc0089c afd12e232ec0 dac75a1cd6c1 2023-07-14T11:11:20.660 INFO:teuthology.orchestra.run.smithi040.stdout:mon.c smithi130 running (7m) 31s ago 7m 63.6M 2048M 18.0.0-4757-g3fc0089c afd12e232ec0 b97a5855a52d 2023-07-14T11:11:20.660 INFO:teuthology.orchestra.run.smithi040.stdout:osd.0 smithi040 running (6m) 21s ago 6m 72.6M 4318M 18.0.0-4757-g3fc0089c afd12e232ec0 46186ae3f891 2023-07-14T11:11:20.660 INFO:teuthology.orchestra.run.smithi040.stdout:osd.1 smithi045 running (6m) 31s ago 6m 77.5M 4293M 18.0.0-4757-g3fc0089c afd12e232ec0 c03d1662225c 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.2 smithi130 running (6m) 31s ago 6m 74.3M 4292M 18.0.0-4757-g3fc0089c afd12e232ec0 d2966549aca2 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.3 smithi040 running (5m) 21s ago 5m 68.0M 4318M 18.0.0-4757-g3fc0089c afd12e232ec0 9ab3b8c0b6fd 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.4 smithi045 running (5m) 31s ago 5m 65.7M 4293M 18.0.0-4757-g3fc0089c afd12e232ec0 c3d6227fb5f1 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.5 smithi130 running (4m) 31s ago 4m 72.2M 4292M 18.0.0-4757-g3fc0089c afd12e232ec0 6b39643af0e6 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.6 smithi040 running (4m) 21s ago 4m 62.1M 4318M 18.0.0-4757-g3fc0089c afd12e232ec0 66ee8171fa96 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.7 smithi045 running (4m) 31s ago 4m 66.1M 4293M 18.0.0-4757-g3fc0089c afd12e232ec0 d7aec47ea497 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.8 smithi130 running (3m) 31s ago 3m 63.4M 4292M 18.0.0-4757-g3fc0089c afd12e232ec0 c508122d7871 2023-07-14T11:11:20.661 INFO:teuthology.orchestra.run.smithi040.stdout:osd.9 smithi040 running (3m) 21s ago 3m 60.8M 4318M 18.0.0-4757-g3fc0089c afd12e232ec0 4041b2bc8254 2023-07-14T11:11:20.662 INFO:teuthology.orchestra.run.smithi040.stdout:osd.10 smithi045 running (2m) 31s ago 2m 61.1M 4293M 18.0.0-4757-g3fc0089c afd12e232ec0 8b23d73bd528 2023-07-14T11:11:20.662 INFO:teuthology.orchestra.run.smithi040.stdout:osd.11 smithi130 running (2m) 31s ago 2m 58.5M 4292M 18.0.0-4757-g3fc0089c afd12e232ec0 da2a953f62c8 2023-07-14T11:11:20.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.656+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 msgr2=0x7f47480a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:20.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.656+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 0x7f47480a3600 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f474c005950 tx=0x7f474c00b410 comp rx=0 tx=0).stop 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 msgr2=0x7f475c1959c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f475800e9a0 tx=0x7f475800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 shutdown_connections 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f47480a1150 0x7f47480a3600 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f475c101720 0x7f475c19cf80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f475c0ffb90 0x7f475c1959c0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 --2- 172.21.15.40:0/456474516 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f475c100d90 0x7f475c195f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:20.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 >> 172.21.15.40:0/456474516 conn(0x7f475c0fb1a0 msgr2=0x7f475c104250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:20.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 shutdown_connections 2023-07-14T11:11:20.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:20.657+0000 7f47642ba700 1 -- 172.21.15.40:0/456474516 wait complete. 2023-07-14T11:11:21.050 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph orch ls' 2023-07-14T11:11:21.788 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:21 smithi130 ceph-mon[129405]: from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:21 smithi130 ceph-mon[129405]: pgmap v329: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:11:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:21 smithi040 ceph-mon[97954]: from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:21 smithi040 ceph-mon[97954]: pgmap v329: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:11:22.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:21 smithi045 ceph-mon[128791]: from='client.15192 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:21 smithi045 ceph-mon[128791]: pgmap v329: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 10 op/s 2023-07-14T11:11:23.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 -- 172.21.15.40:0/2289440158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 msgr2=0x7fc708107fe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 --2- 172.21.15.40:0/2289440158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708107fe0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fc70000b0a0 tx=0x7fc70000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 -- 172.21.15.40:0/2289440158 shutdown_connections 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 --2- 172.21.15.40:0/2289440158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708107fe0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 --2- 172.21.15.40:0/2289440158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 0x7fc7080fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 --2- 172.21.15.40:0/2289440158 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 0x7fc7081011e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 -- 172.21.15.40:0/2289440158 >> 172.21.15.40:0/2289440158 conn(0x7fc7080fb1a0 msgr2=0x7fc7080fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:23.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 -- 172.21.15.40:0/2289440158 shutdown_connections 2023-07-14T11:11:23.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.250+0000 7fc70fec3700 1 -- 172.21.15.40:0/2289440158 wait complete. 2023-07-14T11:11:23.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.251+0000 7fc70fec3700 1 Processor -- start 2023-07-14T11:11:23.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 -- start start 2023-07-14T11:11:23.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 0x7fc7080742b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:23.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 0x7fc708072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:23.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc708199850 con 0x7fc708101720 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc708073380 con 0x7fc708100d90 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70fec3700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc708073540 con 0x7fc7080ffb90 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70e460700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70e460700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42664/0 (socket says 172.21.15.40:42664) 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70e460700 1 -- 172.21.15.40:0/1976797043 learned_addr learned my addr 172.21.15.40:0/1976797043 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:23.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70dc5f700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 0x7fc7080742b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.252+0000 7fc70d45e700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 0x7fc708072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 -- 172.21.15.40:0/1976797043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 msgr2=0x7fc7080742b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 0x7fc7080742b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 -- 172.21.15.40:0/1976797043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 msgr2=0x7fc708072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 0x7fc708072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 -- 172.21.15.40:0/1976797043 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc700009d00 con 0x7fc708101720 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70e460700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fc700009fd0 tx=0x7fc70000bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:23.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70000e040 con 0x7fc708101720 2023-07-14T11:11:23.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7081a4220 con 0x7fc708101720 2023-07-14T11:11:23.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7000094a0 con 0x7fc708101720 2023-07-14T11:11:23.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.253+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc70001b6c0 con 0x7fc708101720 2023-07-14T11:11:23.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.254+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc7081a4580 con 0x7fc708101720 2023-07-14T11:11:23.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.255+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc708107110 con 0x7fc708101720 2023-07-14T11:11:23.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.256+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc70001d470 con 0x7fc708101720 2023-07-14T11:11:23.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.258+0000 7fc6feffd700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 0x7fc6f40a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:23.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.258+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc7000cdec0 con 0x7fc708101720 2023-07-14T11:11:23.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.258+0000 7fc70dc5f700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 0x7fc6f40a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:23.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.259+0000 7fc70dc5f700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 0x7fc6f40a3600 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fc708100bf0 tx=0x7fc704009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:23.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.262+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc7000972f0 con 0x7fc708101720 2023-07-14T11:11:23.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.478+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) v1 -- 0x7fc708060950 con 0x7fc6f40a1150 2023-07-14T11:11:23.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.483+0000 7fc6feffd700 1 -- 172.21.15.40:0/1976797043 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+980 (secure 0 0 0) 0x7fc708060950 con 0x7fc6f40a1150 2023-07-14T11:11:23.485 INFO:teuthology.orchestra.run.smithi040.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2023-07-14T11:11:23.485 INFO:teuthology.orchestra.run.smithi040.stdout:mds.all 12/12 34s ago 2m smithi040=a;smithi040=d;smithi040=g;smithi040=j;smithi045=b;smithi045=e;smithi045=h;smithi045=k;smithi130=c;smithi130=f;smithi130=i;smithi130=l;count:12 2023-07-14T11:11:23.486 INFO:teuthology.orchestra.run.smithi040.stdout:mgr 3/3 34s ago 7m smithi040=x;smithi045=y;smithi130=z;count:3 2023-07-14T11:11:23.486 INFO:teuthology.orchestra.run.smithi040.stdout:mon 3/3 34s ago 7m smithi040:172.21.15.40=a;smithi045:172.21.15.45=b;smithi130:172.21.15.130=c;count:3 2023-07-14T11:11:23.486 INFO:teuthology.orchestra.run.smithi040.stdout:osd 12 34s ago - 2023-07-14T11:11:23.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 msgr2=0x7fc6f40a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:23.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 0x7fc6f40a3600 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7fc708100bf0 tx=0x7fc704009260 comp rx=0 tx=0).stop 2023-07-14T11:11:23.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 msgr2=0x7fc708072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:23.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fc700009fd0 tx=0x7fc70000bce0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 shutdown_connections 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6f40a1150 0x7fc6f40a3600 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc708101720 0x7fc708072e40 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc7080ffb90 0x7fc7080742b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 --2- 172.21.15.40:0/1976797043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc708100d90 0x7fc708072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 >> 172.21.15.40:0/1976797043 conn(0x7fc7080fb1a0 msgr2=0x7fc708104250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 shutdown_connections 2023-07-14T11:11:23.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:23.486+0000 7fc70fec3700 1 -- 172.21.15.40:0/1976797043 wait complete. 2023-07-14T11:11:23.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:23 smithi040 ceph-mon[97954]: pgmap v330: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:23.880 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph orch host ls' 2023-07-14T11:11:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:23 smithi130 ceph-mon[129405]: pgmap v330: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:23 smithi045 ceph-mon[128791]: pgmap v330: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:24.616 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:25.019 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:24 smithi040 ceph-mon[97954]: from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:25.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:24 smithi130 ceph-mon[129405]: from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:25.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:24 smithi045 ceph-mon[128791]: from='client.15198 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 -- 172.21.15.40:0/1393839707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4074540 msgr2=0x7f04f40729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 --2- 172.21.15.40:0/1393839707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4074540 0x7f04f40729a0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f04f0009990 tx=0x7f04f0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 -- 172.21.15.40:0/1393839707 shutdown_connections 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 --2- 172.21.15.40:0/1393839707 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4107a60 0x7f04f4109ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 --2- 172.21.15.40:0/1393839707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4074540 0x7f04f40729a0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 --2- 172.21.15.40:0/1393839707 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 0x7f04f40733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 -- 172.21.15.40:0/1393839707 >> 172.21.15.40:0/1393839707 conn(0x7f04f40fb1a0 msgr2=0x7f04f40fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:26.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 -- 172.21.15.40:0/1393839707 shutdown_connections 2023-07-14T11:11:26.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.018+0000 7f04fc0df700 1 -- 172.21.15.40:0/1393839707 wait complete. 2023-07-14T11:11:26.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.019+0000 7f04fc0df700 1 Processor -- start 2023-07-14T11:11:26.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 -- start start 2023-07-14T11:11:26.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 0x7f04f4197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:26.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4074540 0x7f04f4198000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04f419f650 con 0x7f04f4107a60 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04f419f790 con 0x7f04f4072f70 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fc0df700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f04f419f920 con 0x7f04f4074540 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fa67c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04f967a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4074540 0x7f04f4198000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:26.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fa67c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:50072/0 (socket says 172.21.15.40:50072) 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04f9e7b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 0x7f04f4197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.020+0000 7f04fa67c700 1 -- 172.21.15.40:0/3450423089 learned_addr learned my addr 172.21.15.40:0/3450423089 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 -- 172.21.15.40:0/3450423089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4074540 msgr2=0x7f04f4198000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4074540 0x7f04f4198000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 -- 172.21.15.40:0/3450423089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 msgr2=0x7f04f4197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 0x7f04f4197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 -- 172.21.15.40:0/3450423089 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f04f00095f0 con 0x7f04f4107a60 2023-07-14T11:11:26.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fa67c700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f04ec00ef60 tx=0x7f04ec00ef90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:26.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04ec018070 con 0x7f04f4107a60 2023-07-14T11:11:26.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f04ec00f040 con 0x7f04f4107a60 2023-07-14T11:11:26.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f04f419fba0 con 0x7f04f4107a60 2023-07-14T11:11:26.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.021+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f04ec00ccc0 con 0x7f04f4107a60 2023-07-14T11:11:26.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.022+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04f41a00a0 con 0x7f04f4107a60 2023-07-14T11:11:26.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.023+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f04f4106110 con 0x7f04f4107a60 2023-07-14T11:11:26.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.025+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f04ec003a00 con 0x7f04f4107a60 2023-07-14T11:11:26.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.026+0000 7f04eaffd700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 0x7f04e00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:26.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.026+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f04ec0cc720 con 0x7f04f4107a60 2023-07-14T11:11:26.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.026+0000 7f04f9e7b700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 0x7f04e00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:26.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.027+0000 7f04f9e7b700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 0x7f04e00a3600 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f04f000b2e0 tx=0x7f04f0005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:26.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.030+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f04ec095b50 con 0x7f04f4107a60 2023-07-14T11:11:26.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:25 smithi130 ceph-mon[129405]: pgmap v331: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:26.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:25 smithi040 ceph-mon[97954]: pgmap v331: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:26.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:25 smithi045 ceph-mon[128791]: pgmap v331: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:26.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.244+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) v1 -- 0x7f04f4060950 con 0x7f04e00a1150 2023-07-14T11:11:26.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.246+0000 7f04eaffd700 1 -- 172.21.15.40:0/3450423089 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+191 (secure 0 0 0) 0x7f04f4060950 con 0x7f04e00a1150 2023-07-14T11:11:26.247 INFO:teuthology.orchestra.run.smithi040.stdout:HOST ADDR LABELS STATUS 2023-07-14T11:11:26.248 INFO:teuthology.orchestra.run.smithi040.stdout:smithi040 172.21.15.40 2023-07-14T11:11:26.248 INFO:teuthology.orchestra.run.smithi040.stdout:smithi045 172.21.15.45 2023-07-14T11:11:26.248 INFO:teuthology.orchestra.run.smithi040.stdout:smithi130 172.21.15.130 2023-07-14T11:11:26.248 INFO:teuthology.orchestra.run.smithi040.stdout:3 hosts in cluster 2023-07-14T11:11:26.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 msgr2=0x7f04e00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:26.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 0x7f04e00a3600 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f04f000b2e0 tx=0x7f04f0005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 msgr2=0x7f04f419f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:26.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f04ec00ef60 tx=0x7f04ec00ef90 comp rx=0 tx=0).stop 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 shutdown_connections 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f04e00a1150 0x7f04e00a3600 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04f4107a60 0x7f04f419f080 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f04f4072f70 0x7f04f4197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 --2- 172.21.15.40:0/3450423089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f04f4074540 0x7f04f4198000 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 >> 172.21.15.40:0/3450423089 conn(0x7f04f40fb1a0 msgr2=0x7f04f40fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.248+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 shutdown_connections 2023-07-14T11:11:26.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:26.249+0000 7f04fc0df700 1 -- 172.21.15.40:0/3450423089 wait complete. 2023-07-14T11:11:26.710 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph orch device ls' 2023-07-14T11:11:27.442 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:27.977 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:27 smithi040 ceph-mon[97954]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:27.978 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:27 smithi040 ceph-mon[97954]: pgmap v332: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:11:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:27 smithi130 ceph-mon[129405]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:27 smithi130 ceph-mon[129405]: pgmap v332: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:11:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:27 smithi045 ceph-mon[128791]: from='client.15204 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:27 smithi045 ceph-mon[128791]: pgmap v332: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:11:28.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 -- 172.21.15.40:0/598389838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 msgr2=0x7ff8a40fffa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:28.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 --2- 172.21.15.40:0/598389838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a40fffa0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7ff8a0009990 tx=0x7ff8a0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 -- 172.21.15.40:0/598389838 shutdown_connections 2023-07-14T11:11:28.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 --2- 172.21.15.40:0/598389838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4101720 0x7ff8a4107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 --2- 172.21.15.40:0/598389838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a40fffa0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 --2- 172.21.15.40:0/598389838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4100d90 0x7ff8a41011e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 -- 172.21.15.40:0/598389838 >> 172.21.15.40:0/598389838 conn(0x7ff8a40fb1a0 msgr2=0x7ff8a40fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:28.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 -- 172.21.15.40:0/598389838 shutdown_connections 2023-07-14T11:11:28.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.844+0000 7ff8ab318700 1 -- 172.21.15.40:0/598389838 wait complete. 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 Processor -- start 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 -- start start 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 0x7ff8a4198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 0x7ff8a419f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:28.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a419f680 con 0x7ff8a40ffb90 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a419f7c0 con 0x7ff8a4101720 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.845+0000 7ff8ab318700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff8a419f950 con 0x7ff8a4100d90 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a88b3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 0x7ff8a4198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a98b5700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 0x7ff8a419f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:50092/0 (socket says 172.21.15.40:50092) 2023-07-14T11:11:28.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 -- 172.21.15.40:0/3876019794 learned_addr learned my addr 172.21.15.40:0/3876019794 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a88b3700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 0x7ff8a4198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33568/0 (socket says 172.21.15.40:33568) 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 -- 172.21.15.40:0/3876019794 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 msgr2=0x7ff8a4198030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 0x7ff8a4198030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 -- 172.21.15.40:0/3876019794 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 msgr2=0x7ff8a419f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 0x7ff8a419f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 -- 172.21.15.40:0/3876019794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8a00095f0 con 0x7ff8a40ffb90 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a98b5700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 0x7ff8a419f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:28.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.846+0000 7ff8a90b4700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7ff8a0004740 tx=0x7ff8a0009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.847+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a001d070 con 0x7ff8a40ffb90 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.847+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff8a419fbd0 con 0x7ff8a40ffb90 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.847+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff8a0021410 con 0x7ff8a40ffb90 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.847+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8a0017650 con 0x7ff8a40ffb90 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.847+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff8a419ffe0 con 0x7ff8a40ffb90 2023-07-14T11:11:28.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.848+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff8a4192260 con 0x7ff8a40ffb90 2023-07-14T11:11:28.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.852+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7ff8a0021580 con 0x7ff8a40ffb90 2023-07-14T11:11:28.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.853+0000 7ff89a7fc700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 0x7ff8900a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:28.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.853+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7ff8a00ce750 con 0x7ff8a40ffb90 2023-07-14T11:11:28.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.853+0000 7ff8a88b3700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 0x7ff8900a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:28.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.854+0000 7ff8a88b3700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 0x7ff8900a3600 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7ff894009c60 tx=0x7ff894009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:28.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:28.855+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff8a0097b80 con 0x7ff8a40ffb90 2023-07-14T11:11:29.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.070+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) v1 -- 0x7ff8a4060950 con 0x7ff8900a1150 2023-07-14T11:11:29.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.072+0000 7ff89a7fc700 1 -- 172.21.15.40:0/3876019794 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+492 (secure 0 0 0) 0x7ff8a4060950 con 0x7ff8900a1150 2023-07-14T11:11:29.073 INFO:teuthology.orchestra.run.smithi040.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2023-07-14T11:11:29.074 INFO:teuthology.orchestra.run.smithi040.stdout:smithi040 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330009J400BGN 372G No 3m ago LVM detected, locked 2023-07-14T11:11:29.074 INFO:teuthology.orchestra.run.smithi040.stdout:smithi045 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB75120098480DGN 447G No 2m ago LVM detected, locked 2023-07-14T11:11:29.074 INFO:teuthology.orchestra.run.smithi040.stdout:smithi130 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620500Q1400BGN 372G No 2m ago LVM detected, locked 2023-07-14T11:11:29.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 msgr2=0x7ff8900a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:29.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 0x7ff8900a3600 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7ff894009c60 tx=0x7ff894009260 comp rx=0 tx=0).stop 2023-07-14T11:11:29.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 msgr2=0x7ff8a4197af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:29.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7ff8a0004740 tx=0x7ff8a0009580 comp rx=0 tx=0).stop 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 shutdown_connections 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7ff8900a1150 0x7ff8900a3600 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff8a4101720 0x7ff8a419f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff8a40ffb90 0x7ff8a4197af0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 --2- 172.21.15.40:0/3876019794 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff8a4100d90 0x7ff8a4198030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.074+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 >> 172.21.15.40:0/3876019794 conn(0x7ff8a40fb1a0 msgr2=0x7ff8a41066d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.075+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 shutdown_connections 2023-07-14T11:11:29.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:29.075+0000 7ff8ab318700 1 -- 172.21.15.40:0/3876019794 wait complete. 2023-07-14T11:11:29.536 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-07-14T11:11:29.546 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:11:29.546 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph fs dump' 2023-07-14T11:11:29.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:29 smithi040 ceph-mon[97954]: pgmap v333: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:29 smithi130 ceph-mon[129405]: pgmap v333: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:29 smithi045 ceph-mon[128791]: pgmap v333: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:30.827 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:30 smithi040 ceph-mon[97954]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:31.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:30 smithi130 ceph-mon[129405]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:31.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:30 smithi045 ceph-mon[128791]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:11:31.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:31 smithi040 ceph-mon[97954]: pgmap v334: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:31.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 -- 172.21.15.40:0/244435733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c100d90 msgr2=0x7f754c1011e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 --2- 172.21.15.40:0/244435733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c100d90 0x7f754c1011e0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f7540009990 tx=0x7f7540009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 -- 172.21.15.40:0/244435733 shutdown_connections 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 --2- 172.21.15.40:0/244435733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c101720 0x7f754c107fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 --2- 172.21.15.40:0/244435733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c0fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 --2- 172.21.15.40:0/244435733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c100d90 0x7f754c1011e0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 -- 172.21.15.40:0/244435733 >> 172.21.15.40:0/244435733 conn(0x7f754c0fb1a0 msgr2=0x7f754c0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:31.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 -- 172.21.15.40:0/244435733 shutdown_connections 2023-07-14T11:11:31.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.827+0000 7f7550836700 1 -- 172.21.15.40:0/244435733 wait complete. 2023-07-14T11:11:31.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 Processor -- start 2023-07-14T11:11:31.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 -- start start 2023-07-14T11:11:31.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c1958b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:31.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 0x7f754c195df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:31.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:31.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f754c19d440 con 0x7f754c101720 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f754c19d580 con 0x7f754c100d90 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.828+0000 7f7550836700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f754c19d710 con 0x7f754c0ffb90 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c1958b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754a59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c1958b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33590/0 (socket says 172.21.15.40:33590) 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754a59c700 1 -- 172.21.15.40:0/4271336852 learned_addr learned my addr 172.21.15.40:0/4271336852 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:31.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f7549d9b700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 0x7f754c195df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:50118/0 (socket says 172.21.15.40:50118) 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 -- 172.21.15.40:0/4271336852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 msgr2=0x7f754c1958b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c1958b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 -- 172.21.15.40:0/4271336852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 msgr2=0x7f754c195df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 0x7f754c195df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 -- 172.21.15.40:0/4271336852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75400095f0 con 0x7f754c101720 2023-07-14T11:11:31.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f7549d9b700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 0x7f754c195df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:31.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.829+0000 7f754ad9d700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f7544009f60 tx=0x7f7544009f90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:31.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.830+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7544018070 con 0x7f754c101720 2023-07-14T11:11:31.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.830+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f754400f040 con 0x7f754c101720 2023-07-14T11:11:31.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.830+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f754c19d990 con 0x7f754c101720 2023-07-14T11:11:31.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.830+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f754400ccc0 con 0x7f754c101720 2023-07-14T11:11:31.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.830+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f754c19de90 con 0x7f754c101720 2023-07-14T11:11:31.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.831+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f754c10b3a0 con 0x7f754c101720 2023-07-14T11:11:31.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.833+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f7544003a00 con 0x7f754c101720 2023-07-14T11:11:31.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.834+0000 7f753b7fe700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 0x7f75340a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:31.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.835+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f75440cc6e0 con 0x7f754c101720 2023-07-14T11:11:31.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.835+0000 7f754a59c700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 0x7f75340a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:31.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.835+0000 7f754a59c700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 0x7f75340a3680 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f753c005950 tx=0x7f753c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:31.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:31.839+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f7544095b10 con 0x7f754c101720 2023-07-14T11:11:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:31 smithi130 ceph-mon[129405]: pgmap v334: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:31 smithi045 ceph-mon[128791]: pgmap v334: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:32.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.089+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f754c10b590 con 0x7f754c101720 2023-07-14T11:11:32.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.090+0000 7f753b7fe700 1 -- 172.21.15.40:0/4271336852 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 27 v27) v1 ==== 76+0+2927 (secure 0 0 0) 0x7f7544099130 con 0x7f754c101720 2023-07-14T11:11:32.092 INFO:teuthology.orchestra.run.smithi040.stdout:e27 2023-07-14T11:11:32.092 INFO:teuthology.orchestra.run.smithi040.stdout:enable_multiple, ever_enabled_multiple: 1,1 2023-07-14T11:11:32.092 INFO:teuthology.orchestra.run.smithi040.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-07-14T11:11:32.092 INFO:teuthology.orchestra.run.smithi040.stdout:legacy client fscid: 1 2023-07-14T11:11:32.092 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:Filesystem 'cephfs' (1) 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:fs_name cephfs 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:epoch 27 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:created 2023-07-14T11:09:43.127642+0000 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:modified 2023-07-14T11:10:17.374269+0000 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:tableserver 0 2023-07-14T11:11:32.093 INFO:teuthology.orchestra.run.smithi040.stdout:root 0 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:session_timeout 300 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:session_autoclose 300 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:max_file_size 1099511627776 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:max_xattr_size 65536 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:required_client_features {} 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:last_failure 0 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:last_failure_osd_epoch 0 2023-07-14T11:11:32.094 INFO:teuthology.orchestra.run.smithi040.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:max_mds 5 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:in 0,1,2,3,4 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:up {0=24569,1=24565,2=24523,3=24521,4=24511} 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:failed 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:damaged 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:stopped 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:data_pools [3,4] 2023-07-14T11:11:32.095 INFO:teuthology.orchestra.run.smithi040.stdout:metadata_pool 2 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:inline_data disabled 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:balancer 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:bal_rank_mask -1 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:standby_count_wanted 0 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.b{0:24569} state up:active seq 3 addr [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.g{0:24553} state up:standby-replay seq 1 addr [v2:172.21.15.40:6838/3591632989,v1:172.21.15.40:6839/3591632989] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.096 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.a{1:24565} state up:active seq 11 addr [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.i{1:24541} state up:standby-replay seq 1 addr [v2:172.21.15.130:6838/3942717247,v1:172.21.15.130:6839/3942717247] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.c{2:24523} state up:active seq 14 addr [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.e{2:24494} state up:standby-replay seq 1 addr [v2:172.21.15.45:6834/1717726567,v1:172.21.15.45:6835/1717726567] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.h{3:24521} state up:active seq 14 addr [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.d{3:24482} state up:standby-replay seq 1 addr [v2:172.21.15.40:6836/625677299,v1:172.21.15.40:6837/625677299] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.f{4:24511} state up:active seq 15 addr [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.l{4:24466} state up:standby-replay seq 1 addr [v2:172.21.15.130:6832/2180852091,v1:172.21.15.130:6833/2180852091] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.097 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:32.098 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:32.098 INFO:teuthology.orchestra.run.smithi040.stdout:Standby daemons: 2023-07-14T11:11:32.098 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:32.098 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.j{-1:14484} state up:standby seq 1 addr [v2:172.21.15.40:6834/2807710685,v1:172.21.15.40:6835/2807710685] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.098 INFO:teuthology.orchestra.run.smithi040.stdout:[mds.k{-1:24461} state up:standby seq 1 addr [v2:172.21.15.45:6832/2674461503,v1:172.21.15.45:6833/2674461503] compat {c=[1],r=[1],i=[7ff]}] 2023-07-14T11:11:32.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 msgr2=0x7f75340a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:32.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 0x7f75340a3680 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f753c005950 tx=0x7f753c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:32.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 msgr2=0x7f754c19ce70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:32.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f7544009f60 tx=0x7f7544009f90 comp rx=0 tx=0).stop 2023-07-14T11:11:32.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 shutdown_connections 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f75340a11d0 0x7f75340a3680 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f754c101720 0x7f754c19ce70 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f754c0ffb90 0x7f754c1958b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 --2- 172.21.15.40:0/4271336852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f754c100d90 0x7f754c195df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 >> 172.21.15.40:0/4271336852 conn(0x7f754c0fb1a0 msgr2=0x7f754c0fd490 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.092+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 shutdown_connections 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:32.093+0000 7f7550836700 1 -- 172.21.15.40:0/4271336852 wait complete. 2023-07-14T11:11:32.100 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:32.583 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- bash -c 'ceph osd dump' 2023-07-14T11:11:33.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:32 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4271336852' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2023-07-14T11:11:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:32 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4271336852' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2023-07-14T11:11:33.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:32 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4271336852' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2023-07-14T11:11:33.323 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:33.835 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:33 smithi040 ceph-mon[97954]: pgmap v335: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:33 smithi130 ceph-mon[129405]: pgmap v335: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:33 smithi045 ceph-mon[128791]: pgmap v335: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 -- 172.21.15.40:0/3178836984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 msgr2=0x7f4cbc0729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 --2- 172.21.15.40:0/3178836984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc0729a0 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f4cb8009990 tx=0x7f4cb8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 -- 172.21.15.40:0/3178836984 shutdown_connections 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 --2- 172.21.15.40:0/3178836984 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc107a60 0x7f4cbc109ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 --2- 172.21.15.40:0/3178836984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc0729a0 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 --2- 172.21.15.40:0/3178836984 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc072f70 0x7f4cbc0733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.898+0000 7f4cc4252700 1 -- 172.21.15.40:0/3178836984 >> 172.21.15.40:0/3178836984 conn(0x7f4cbc0fb1a0 msgr2=0x7f4cbc0fd610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.899+0000 7f4cc4252700 1 -- 172.21.15.40:0/3178836984 shutdown_connections 2023-07-14T11:11:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.899+0000 7f4cc4252700 1 -- 172.21.15.40:0/3178836984 wait complete. 2023-07-14T11:11:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 Processor -- start 2023-07-14T11:11:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 -- start start 2023-07-14T11:11:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc072f70 0x7f4cbc197ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc107a60 0x7f4cbc19f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cbc19f660 con 0x7f4cbc074540 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cbc19f7a0 con 0x7f4cbc072f70 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc4252700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cbc19f930 con 0x7f4cbc107a60 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc17ed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc17ed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39492/0 (socket says 172.21.15.40:39492) 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.900+0000 7f4cc17ed700 1 -- 172.21.15.40:0/2505528991 learned_addr learned my addr 172.21.15.40:0/2505528991 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc1fee700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc072f70 0x7f4cbc197ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc27ef700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc107a60 0x7f4cbc19f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 -- 172.21.15.40:0/2505528991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc107a60 msgr2=0x7f4cbc19f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc107a60 0x7f4cbc19f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 -- 172.21.15.40:0/2505528991 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc072f70 msgr2=0x7f4cbc197ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc072f70 0x7f4cbc197ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 -- 172.21.15.40:0/2505528991 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cb80095f0 con 0x7f4cbc074540 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc17ed700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f4cac00e9d0 tx=0x7f4cac00ece0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cac009790 con 0x7f4cbc074540 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4cac00ee70 con 0x7f4cbc074540 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cbc19fc10 con 0x7f4cbc074540 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.901+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cac010ea0 con 0x7f4cbc074540 2023-07-14T11:11:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.902+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cbc1a0160 con 0x7f4cbc074540 2023-07-14T11:11:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.903+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4cbc106110 con 0x7f4cbc074540 2023-07-14T11:11:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.904+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4cac00cbd0 con 0x7f4cbc074540 2023-07-14T11:11:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.906+0000 7f4cb2ffd700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 0x7f4ca80a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.906+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4cac014070 con 0x7f4cbc074540 2023-07-14T11:11:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.906+0000 7f4cc1fee700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 0x7f4ca80a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.907+0000 7f4cc1fee700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 0x7f4ca80a3680 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f4cb800b2e0 tx=0x7f4cb8005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:34.910+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4cac051790 con 0x7f4cbc074540 2023-07-14T11:11:35.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.118+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump"} v 0) v1 -- 0x7f4cbc02c850 con 0x7f4cbc074540 2023-07-14T11:11:35.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.119+0000 7f4cb2ffd700 1 -- 172.21.15.40:0/2505528991 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump"}]=0 v92) v1 ==== 56+0+5635 (secure 0 0 0) 0x7f4cac096190 con 0x7f4cbc074540 2023-07-14T11:11:35.121 INFO:teuthology.orchestra.run.smithi040.stdout:epoch 92 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:fsid bbe0b002-2235-11ee-9b32-001a4aab830c 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:created 2023-07-14T11:02:08.998735+0000 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:modified 2023-07-14T11:11:08.711561+0000 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:crush_version 27 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:full_ratio 0.95 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:backfillfull_ratio 0.9 2023-07-14T11:11:35.122 INFO:teuthology.orchestra.run.smithi040.stdout:nearfull_ratio 0.85 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:require_min_compat_client luminous 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:min_compat_client luminous 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:require_osd_release reef 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:stretch_mode_enabled false 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 12.00 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:pool 2 'cephfs_metadata' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 83 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.25 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:pool 3 'cephfs_data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 64 pgp_num 64 autoscale_mode off last_change 84 flags hashpspool stripe_width 0 pg_num_min 64 target_size_ratio 0.05 application cephfs read_balance_score 1.50 2023-07-14T11:11:35.123 INFO:teuthology.orchestra.run.smithi040.stdout:pool 4 'cephfs_data_ec' erasure profile cephfs_data_ec size 4 min_size 3 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 88 flags hashpspool,ec_overwrites stripe_width 8192 pg_num_min 64 target_size_ratio 0.9 application cephfs 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:max_osd 12 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.0 up in weight 1 up_from 8 up_thru 86 down_at 0 last_clean_interval [0,0) [v2:172.21.15.40:6802/2649857957,v1:172.21.15.40:6803/2649857957] [v2:172.21.15.40:6804/2649857957,v1:172.21.15.40:6805/2649857957] exists,up ec960268-07b9-478c-9217-22b4c81f02b2 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.1 up in weight 1 up_from 13 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6800/21717094,v1:172.21.15.45:6801/21717094] [v2:172.21.15.45:6802/21717094,v1:172.21.15.45:6803/21717094] exists,up 6d1f5898-462a-4229-8a58-4c31ab9a6b13 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.2 up in weight 1 up_from 18 up_thru 86 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6800/1035271032,v1:172.21.15.130:6801/1035271032] [v2:172.21.15.130:6802/1035271032,v1:172.21.15.130:6803/1035271032] exists,up 6a098e7e-81d5-4e1d-b613-ff144e76f342 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.3 up in weight 1 up_from 25 up_thru 86 down_at 0 last_clean_interval [0,0) [v2:172.21.15.40:6810/741691351,v1:172.21.15.40:6811/741691351] [v2:172.21.15.40:6812/741691351,v1:172.21.15.40:6813/741691351] exists,up d065670e-a11a-4f71-a845-1059f16de6bd 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.4 up in weight 1 up_from 33 up_thru 86 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6808/2253566846,v1:172.21.15.45:6809/2253566846] [v2:172.21.15.45:6810/2253566846,v1:172.21.15.45:6811/2253566846] exists,up 6fe941ce-8407-48e2-ac92-bedcf87816b3 2023-07-14T11:11:35.124 INFO:teuthology.orchestra.run.smithi040.stdout:osd.5 up in weight 1 up_from 38 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6808/3826515733,v1:172.21.15.130:6809/3826515733] [v2:172.21.15.130:6810/3826515733,v1:172.21.15.130:6811/3826515733] exists,up 8f6be8ae-e857-4175-b9ea-9da1eeabcfd0 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.6 up in weight 1 up_from 46 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.40:6818/4118960469,v1:172.21.15.40:6819/4118960469] [v2:172.21.15.40:6820/4118960469,v1:172.21.15.40:6821/4118960469] exists,up 04fdd37c-4112-429a-b73e-7ff7d0da5177 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.7 up in weight 1 up_from 51 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6816/2329017282,v1:172.21.15.45:6817/2329017282] [v2:172.21.15.45:6818/2329017282,v1:172.21.15.45:6819/2329017282] exists,up 50a6c695-d861-4a14-8473-06d9c6d991d6 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.8 up in weight 1 up_from 59 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6816/1753746487,v1:172.21.15.130:6817/1753746487] [v2:172.21.15.130:6818/1753746487,v1:172.21.15.130:6819/1753746487] exists,up 2fcfa29f-64c0-450c-ba0d-2a74a38550bb 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.9 up in weight 1 up_from 67 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.40:6826/1901957249,v1:172.21.15.40:6827/1901957249] [v2:172.21.15.40:6828/1901957249,v1:172.21.15.40:6829/1901957249] exists,up f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.10 up in weight 1 up_from 72 up_thru 91 down_at 0 last_clean_interval [0,0) [v2:172.21.15.45:6824/3539261425,v1:172.21.15.45:6825/3539261425] [v2:172.21.15.45:6826/3539261425,v1:172.21.15.45:6827/3539261425] exists,up 8fb75cd9-7dc5-4825-b957-ebe7b33c49c7 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:osd.11 up in weight 1 up_from 79 up_thru 89 down_at 0 last_clean_interval [0,0) [v2:172.21.15.130:6824/692245935,v1:172.21.15.130:6825/692245935] [v2:172.21.15.130:6826/692245935,v1:172.21.15.130:6827/692245935] exists,up ccf993ad-d29e-44b7-9aa1-bf856fc9a92e 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 2.0 [8,1] 2023-07-14T11:11:35.125 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 2.6 [8,11] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 2.8 [2,11] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 2.e [2,4] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 3.14 [4,6] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 3.1b [4,8] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 3.24 [4,9] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 3.25 [5,9] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 3.27 [3,9] 2023-07-14T11:11:35.126 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 4.2 [9,1] 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 4.12 [9,10,5,2] 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:pg_upmap_items 4.14 [9,8] 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:6801/2376955220 expires 2023-07-15T11:03:06.541389+0000 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/3160212490 expires 2023-07-15T11:03:06.541389+0000 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/4083556243 expires 2023-07-15T11:03:06.541389+0000 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/2326174215 expires 2023-07-15T11:02:39.441675+0000 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/1897779716 expires 2023-07-15T11:03:06.541389+0000 2023-07-14T11:11:35.127 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/3378800182 expires 2023-07-15T11:02:39.441675+0000 2023-07-14T11:11:35.128 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:6800/137331595 expires 2023-07-15T11:02:39.441675+0000 2023-07-14T11:11:35.128 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:6801/137331595 expires 2023-07-15T11:02:39.441675+0000 2023-07-14T11:11:35.128 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:6800/2376955220 expires 2023-07-15T11:03:06.541389+0000 2023-07-14T11:11:35.128 INFO:teuthology.orchestra.run.smithi040.stdout:blocklist 172.21.15.40:0/500170929 expires 2023-07-15T11:02:39.441675+0000 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 msgr2=0x7f4ca80a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 0x7f4ca80a3680 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f4cb800b2e0 tx=0x7f4cb8005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 msgr2=0x7f4cbc198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f4cac00e9d0 tx=0x7f4cac00ece0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 shutdown_connections 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4ca80a11d0 0x7f4ca80a3680 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cbc107a60 0x7f4cbc19f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cbc072f70 0x7f4cbc197ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 --2- 172.21.15.40:0/2505528991 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cbc074540 0x7f4cbc198010 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 >> 172.21.15.40:0/2505528991 conn(0x7f4cbc0fb1a0 msgr2=0x7f4cbc0fd580 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:35.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 shutdown_connections 2023-07-14T11:11:35.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.122+0000 7f4cc4252700 1 -- 172.21.15.40:0/2505528991 wait complete. 2023-07-14T11:11:35.514 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:35 smithi040 ceph-mon[97954]: pgmap v336: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:35.514 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:35 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2505528991' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-07-14T11:11:35.517 INFO:teuthology.run_tasks:Running task fs.ready... 2023-07-14T11:11:35.528 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:35.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.756+0000 7f17b91ff700 1 -- 172.21.15.40:0/1079210846 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4074550 msgr2=0x7f17b40729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.756+0000 7f17b91ff700 1 --2- 172.21.15.40:0/1079210846 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4074550 0x7f17b40729a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f17a40098f0 tx=0x7f17a4009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 -- 172.21.15.40:0/1079210846 shutdown_connections 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 --2- 172.21.15.40:0/1079210846 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b41079c0 0x7f17b4109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 --2- 172.21.15.40:0/1079210846 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4074550 0x7f17b40729a0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 --2- 172.21.15.40:0/1079210846 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b4072f70 0x7f17b40733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 -- 172.21.15.40:0/1079210846 >> 172.21.15.40:0/1079210846 conn(0x7f17b40fb0e0 msgr2=0x7f17b40fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 -- 172.21.15.40:0/1079210846 shutdown_connections 2023-07-14T11:11:35.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.757+0000 7f17b91ff700 1 -- 172.21.15.40:0/1079210846 wait complete. 2023-07-14T11:11:35.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.758+0000 7f17b91ff700 1 Processor -- start 2023-07-14T11:11:35.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.758+0000 7f17b91ff700 1 -- start start 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b4074550 0x7f17b4197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b41079c0 0x7f17b419f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b419f630 con 0x7f17b4072f70 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b419f770 con 0x7f17b41079c0 2023-07-14T11:11:35.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b91ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f17b419f900 con 0x7f17b4074550 2023-07-14T11:11:35.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b2d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:35.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b2d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39524/0 (socket says 172.21.15.40:39524) 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b2d9d700 1 -- 172.21.15.40:0/3691453815 learned_addr learned my addr 172.21.15.40:0/3691453815 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b359e700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b41079c0 0x7f17b419f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.759+0000 7f17b259c700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b4074550 0x7f17b4197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 -- 172.21.15.40:0/3691453815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b4074550 msgr2=0x7f17b4197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b4074550 0x7f17b4197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 -- 172.21.15.40:0/3691453815 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b41079c0 msgr2=0x7f17b419f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b41079c0 0x7f17b419f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 -- 172.21.15.40:0/3691453815 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17a4009520 con 0x7f17b4072f70 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b2d9d700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f17a4009230 tx=0x7f17a400b790 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17a401c070 con 0x7f17b4072f70 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f17b419fb80 con 0x7f17b4072f70 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f17a400f860 con 0x7f17b4072f70 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f17a4003a00 con 0x7f17b4072f70 2023-07-14T11:11:35.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.760+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17b41a0000 con 0x7f17b4072f70 2023-07-14T11:11:35.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.761+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f17b4106070 con 0x7f17b4072f70 2023-07-14T11:11:35.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.763+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f17a4020030 con 0x7f17b4072f70 2023-07-14T11:11:35.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.765+0000 7f179bfff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 0x7f179c0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:35.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.765+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f17a40ce160 con 0x7f17b4072f70 2023-07-14T11:11:35.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.765+0000 7f17b259c700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 0x7f179c0a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:35.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.766+0000 7f17b259c700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 0x7f179c0a3650 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f17b4074270 tx=0x7f17a8009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:35.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.766+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f17a4097590 con 0x7f17b4072f70 2023-07-14T11:11:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:35 smithi130 ceph-mon[129405]: pgmap v336: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:35 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2505528991' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-07-14T11:11:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:35 smithi045 ceph-mon[128791]: pgmap v336: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:35 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2505528991' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-07-14T11:11:35.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.977+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f17b402c850 con 0x7f17b4072f70 2023-07-14T11:11:35.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.979+0000 7f179bfff700 1 -- 172.21.15.40:0/3691453815 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f17a4095e60 con 0x7f17b4072f70 2023-07-14T11:11:35.981 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:35.981 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:35.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 msgr2=0x7f179c0a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 0x7f179c0a3650 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f17b4074270 tx=0x7f17a8009310 comp rx=0 tx=0).stop 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 msgr2=0x7f17b4197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f17a4009230 tx=0x7f17a400b790 comp rx=0 tx=0).stop 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 shutdown_connections 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f179c0a11a0 0x7f179c0a3650 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f17b41079c0 0x7f17b419f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17b4072f70 0x7f17b4197ab0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 --2- 172.21.15.40:0/3691453815 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17b4074550 0x7f17b4197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:35.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 >> 172.21.15.40:0/3691453815 conn(0x7f17b40fb0e0 msgr2=0x7f17b40fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:35.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 shutdown_connections 2023-07-14T11:11:35.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:35.981+0000 7f17b91ff700 1 -- 172.21.15.40:0/3691453815 wait complete. 2023-07-14T11:11:36.001 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:36.002 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:36.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 -- 172.21.15.40:0/3474591122 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf500ffae0 msgr2=0x7fdf500ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 --2- 172.21.15.40:0/3474591122 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf500ffae0 0x7fdf500ffef0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fdf40009990 tx=0x7fdf40009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 -- 172.21.15.40:0/3474591122 shutdown_connections 2023-07-14T11:11:36.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 --2- 172.21.15.40:0/3474591122 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 0x7fdf50107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 --2- 172.21.15.40:0/3474591122 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf500ffae0 0x7fdf500ffef0 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 --2- 172.21.15.40:0/3474591122 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf50100cd0 0x7fdf50101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 -- 172.21.15.40:0/3474591122 >> 172.21.15.40:0/3474591122 conn(0x7fdf500fb0e0 msgr2=0x7fdf500fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:36.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 -- 172.21.15.40:0/3474591122 shutdown_connections 2023-07-14T11:11:36.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.207+0000 7fdf55fff700 1 -- 172.21.15.40:0/3474591122 wait complete. 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.208+0000 7fdf55fff700 1 Processor -- start 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 -- start start 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 0x7fdf50197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 0x7fdf5019f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf5019f600 con 0x7fdf50100cd0 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf5019f740 con 0x7fdf50101660 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf55fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdf5019f8d0 con 0x7fdf500ffae0 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf4effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf4f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 0x7fdf50197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf4effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39540/0 (socket says 172.21.15.40:39540) 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf4effd700 1 -- 172.21.15.40:0/1409474040 learned_addr learned my addr 172.21.15.40:0/1409474040 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.209+0000 7fdf4ffff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 0x7fdf5019f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 -- 172.21.15.40:0/1409474040 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 msgr2=0x7fdf5019f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 0x7fdf5019f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 -- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 msgr2=0x7fdf50197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 -- 172.21.15.40:0/1409474040 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdf400095f0 con 0x7fdf500ffae0 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4effd700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4f7fe700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 0x7fdf50197a80 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fdf4000b620 tx=0x7fdf40004730 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf4001d070 con 0x7fdf500ffae0 2023-07-14T11:11:36.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdf5019fb50 con 0x7fdf500ffae0 2023-07-14T11:11:36.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdf40021410 con 0x7fdf500ffae0 2023-07-14T11:11:36.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.210+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdf40017650 con 0x7fdf500ffae0 2023-07-14T11:11:36.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.211+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdf5019ffd0 con 0x7fdf500ffae0 2023-07-14T11:11:36.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.211+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdf5010b300 con 0x7fdf500ffae0 2023-07-14T11:11:36.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.214+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fdf400217f0 con 0x7fdf500ffae0 2023-07-14T11:11:36.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.215+0000 7fdf4cff9700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 0x7fdf380a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.215+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fdf400ce540 con 0x7fdf500ffae0 2023-07-14T11:11:36.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.215+0000 7fdf4effd700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 0x7fdf380a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.216+0000 7fdf4effd700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 0x7fdf380a3680 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fdf44011550 tx=0x7fdf44009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:36.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.217+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdf40097970 con 0x7fdf500ffae0 2023-07-14T11:11:36.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.428+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fdf501a0580 con 0x7fdf500ffae0 2023-07-14T11:11:36.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.429+0000 7fdf4cff9700 1 -- 172.21.15.40:0/1409474040 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7fdf40021e60 con 0x7fdf500ffae0 2023-07-14T11:11:36.431 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:36.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 msgr2=0x7fdf380a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 0x7fdf380a3680 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fdf44011550 tx=0x7fdf44009260 comp rx=0 tx=0).stop 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 msgr2=0x7fdf50197a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 0x7fdf50197a80 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7fdf4000b620 tx=0x7fdf40004730 comp rx=0 tx=0).stop 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 shutdown_connections 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdf380a11d0 0x7fdf380a3680 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdf50101660 0x7fdf5019f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdf500ffae0 0x7fdf50197a80 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 --2- 172.21.15.40:0/1409474040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdf50100cd0 0x7fdf50197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 >> 172.21.15.40:0/1409474040 conn(0x7fdf500fb0e0 msgr2=0x7fdf500fd360 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:36.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 shutdown_connections 2023-07-14T11:11:36.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.432+0000 7fdf55fff700 1 -- 172.21.15.40:0/1409474040 wait complete. 2023-07-14T11:11:36.452 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:36.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 -- 172.21.15.40:0/1044910488 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 msgr2=0x7f56f00fff00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 --2- 172.21.15.40:0/1044910488 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f00fff00 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f56ec009990 tx=0x7f56ec009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 -- 172.21.15.40:0/1044910488 shutdown_connections 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 --2- 172.21.15.40:0/1044910488 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0101670 0x7f56f0107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 --2- 172.21.15.40:0/1044910488 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f00fff00 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 --2- 172.21.15.40:0/1044910488 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0100ce0 0x7f56f0101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 -- 172.21.15.40:0/1044910488 >> 172.21.15.40:0/1044910488 conn(0x7f56f00fb130 msgr2=0x7f56f00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.658+0000 7f56f7dff700 1 -- 172.21.15.40:0/1044910488 shutdown_connections 2023-07-14T11:11:36.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.659+0000 7f56f7dff700 1 -- 172.21.15.40:0/1044910488 wait complete. 2023-07-14T11:11:36.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.659+0000 7f56f7dff700 1 Processor -- start 2023-07-14T11:11:36.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 -- start start 2023-07-14T11:11:36.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 0x7f56f0197fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0101670 0x7f56f019f050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f019f620 con 0x7f56f00ffaf0 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f019f760 con 0x7f56f0100ce0 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f7dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f019f8f0 con 0x7f56f0101670 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f5b9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f5b9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39568/0 (socket says 172.21.15.40:39568) 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f5b9b700 1 -- 172.21.15.40:0/3566910524 learned_addr learned my addr 172.21.15.40:0/3566910524 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f639c700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0101670 0x7f56f019f050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.660+0000 7f56f539a700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 0x7f56f0197fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 -- 172.21.15.40:0/3566910524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0101670 msgr2=0x7f56f019f050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0101670 0x7f56f019f050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 -- 172.21.15.40:0/3566910524 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 msgr2=0x7f56f0197fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 0x7f56f0197fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 -- 172.21.15.40:0/3566910524 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56ec0095f0 con 0x7f56f00ffaf0 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f539a700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 0x7f56f0197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f5b9b700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f56ec004730 tx=0x7f56ec004760 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:36.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56ec01d070 con 0x7f56f00ffaf0 2023-07-14T11:11:36.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56f019fb70 con 0x7f56f00ffaf0 2023-07-14T11:11:36.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f56ec021410 con 0x7f56f00ffaf0 2023-07-14T11:11:36.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f56ec017650 con 0x7f56f00ffaf0 2023-07-14T11:11:36.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.661+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56f019fff0 con 0x7f56f00ffaf0 2023-07-14T11:11:36.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.662+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56f01921d0 con 0x7f56f00ffaf0 2023-07-14T11:11:36.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.664+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f56ec021810 con 0x7f56f00ffaf0 2023-07-14T11:11:36.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.666+0000 7f56e6ffd700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 0x7f56dc0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:36.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.666+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f56ec0ce7f0 con 0x7f56f00ffaf0 2023-07-14T11:11:36.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.666+0000 7f56f539a700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 0x7f56dc0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:36.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.666+0000 7f56f539a700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 0x7f56dc0a3600 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f56e0009cb0 tx=0x7f56e0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:36.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.668+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f56ec097c20 con 0x7f56f00ffaf0 2023-07-14T11:11:36.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:36 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3691453815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:36.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:36 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3691453815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:36.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:36 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3691453815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.877+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f56f002c9e0 con 0x7f56f00ffaf0 2023-07-14T11:11:36.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.877+0000 7f56e6ffd700 1 -- 172.21.15.40:0/3566910524 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7f56ec09b240 con 0x7f56f00ffaf0 2023-07-14T11:11:36.879 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:36.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 msgr2=0x7f56dc0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 0x7f56dc0a3600 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f56e0009cb0 tx=0x7f56e0009260 comp rx=0 tx=0).stop 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 msgr2=0x7f56f0197aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f56ec004730 tx=0x7f56ec004760 comp rx=0 tx=0).stop 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 shutdown_connections 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f56dc0a1150 0x7f56dc0a3600 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f56f0101670 0x7f56f019f050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56f00ffaf0 0x7f56f0197aa0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 --2- 172.21.15.40:0/3566910524 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f56f0100ce0 0x7f56f0197fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:36.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 >> 172.21.15.40:0/3566910524 conn(0x7f56f00fb130 msgr2=0x7f56f01065b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:36.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 shutdown_connections 2023-07-14T11:11:36.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:36.880+0000 7f56f7dff700 1 -- 172.21.15.40:0/3566910524 wait complete. 2023-07-14T11:11:36.900 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 -- 172.21.15.40:0/37872831 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 msgr2=0x7f666c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 --2- 172.21.15.40:0/37872831 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c101120 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f6660009990 tx=0x7f6660009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 -- 172.21.15.40:0/37872831 shutdown_connections 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 --2- 172.21.15.40:0/37872831 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c101660 0x7f666c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 --2- 172.21.15.40:0/37872831 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c0ffae0 0x7f666c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 --2- 172.21.15.40:0/37872831 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c101120 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 -- 172.21.15.40:0/37872831 >> 172.21.15.40:0/37872831 conn(0x7f666c0fb0e0 msgr2=0x7f666c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:37.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 -- 172.21.15.40:0/37872831 shutdown_connections 2023-07-14T11:11:37.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.105+0000 7f66725ff700 1 -- 172.21.15.40:0/37872831 wait complete. 2023-07-14T11:11:37.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.106+0000 7f66725ff700 1 Processor -- start 2023-07-14T11:11:37.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.106+0000 7f66725ff700 1 -- start start 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c101660 0x7f666c19f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f666c19f5f0 con 0x7f666c0ffae0 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f666c19f730 con 0x7f666c100cd0 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f66725ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f666c19f8c0 con 0x7f666c101660 2023-07-14T11:11:37.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39596/0 (socket says 172.21.15.40:39596) 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 -- 172.21.15.40:0/1819614121 learned_addr learned my addr 172.21.15.40:0/1819614121 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666b7fe700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f6670b9c700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c101660 0x7f666c19f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 -- 172.21.15.40:0/1819614121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c101660 msgr2=0x7f666c19f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c101660 0x7f666c19f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 -- 172.21.15.40:0/1819614121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 msgr2=0x7f666c197fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c197fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666bfff700 1 -- 172.21.15.40:0/1819614121 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f66600095f0 con 0x7f666c0ffae0 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.107+0000 7f666b7fe700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f666bfff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f665c00e970 tx=0x7f665c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f665c009790 con 0x7f666c0ffae0 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f665c00eec0 con 0x7f666c0ffae0 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f665c003cf0 con 0x7f666c0ffae0 2023-07-14T11:11:37.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f666c19fba0 con 0x7f666c0ffae0 2023-07-14T11:11:37.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.108+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f666c1a0060 con 0x7f666c0ffae0 2023-07-14T11:11:37.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.109+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f666c1921c0 con 0x7f666c0ffae0 2023-07-14T11:11:37.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.111+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f665c00ccd0 con 0x7f666c0ffae0 2023-07-14T11:11:37.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.113+0000 7f66697fa700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 0x7f66540a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.113+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f665c014070 con 0x7f666c0ffae0 2023-07-14T11:11:37.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.113+0000 7f666b7fe700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 0x7f66540a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.114+0000 7f666b7fe700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 0x7f66540a3600 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f666000b2e0 tx=0x7f6660005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:37.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.117+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f665c0962b0 con 0x7f666c0ffae0 2023-07-14T11:11:37.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.325+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f666c0611c0 con 0x7f666c0ffae0 2023-07-14T11:11:37.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.326+0000 7f66697fa700 1 -- 172.21.15.40:0/1819614121 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7f665c0998d0 con 0x7f666c0ffae0 2023-07-14T11:11:37.327 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 msgr2=0x7f66540a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 0x7f66540a3600 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f666000b2e0 tx=0x7f6660005fd0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 msgr2=0x7f666c197a70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f665c00e970 tx=0x7f665c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 shutdown_connections 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f66540a1150 0x7f66540a3600 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f666c101660 0x7f666c19f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f666c0ffae0 0x7f666c197a70 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 --2- 172.21.15.40:0/1819614121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f666c100cd0 0x7f666c197fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 >> 172.21.15.40:0/1819614121 conn(0x7f666c0fb0e0 msgr2=0x7f666c1065a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.328+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 shutdown_connections 2023-07-14T11:11:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.329+0000 7f66725ff700 1 -- 172.21.15.40:0/1819614121 wait complete. 2023-07-14T11:11:37.349 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:37.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 -- 172.21.15.40:0/578169010 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 msgr2=0x7fba18107f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 --2- 172.21.15.40:0/578169010 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18107f30 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fba1000b0a0 tx=0x7fba1000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 -- 172.21.15.40:0/578169010 shutdown_connections 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 --2- 172.21.15.40:0/578169010 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18107f30 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 --2- 172.21.15.40:0/578169010 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba180ffaf0 0x7fba180fff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 --2- 172.21.15.40:0/578169010 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba18100ce0 0x7fba18101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 -- 172.21.15.40:0/578169010 >> 172.21.15.40:0/578169010 conn(0x7fba180fb130 msgr2=0x7fba180fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 -- 172.21.15.40:0/578169010 shutdown_connections 2023-07-14T11:11:37.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.554+0000 7fba1cdff700 1 -- 172.21.15.40:0/578169010 wait complete. 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.555+0000 7fba1cdff700 1 Processor -- start 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.555+0000 7fba1cdff700 1 -- start start 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba18100ce0 0x7fba18072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba180735e0 con 0x7fba18101670 2023-07-14T11:11:37.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba18073750 con 0x7fba180ffaf0 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1cdff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba1819f9c0 con 0x7fba18100ce0 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba16d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54680/0 (socket says 172.21.15.40:54680) 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba1659c700 1 -- 172.21.15.40:0/3329891275 learned_addr learned my addr 172.21.15.40:0/3329891275 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba15d9b700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba18100ce0 0x7fba18072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba16d9d700 1 -- 172.21.15.40:0/3329891275 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba18100ce0 msgr2=0x7fba18072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba16d9d700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba18100ce0 0x7fba18072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.556+0000 7fba16d9d700 1 -- 172.21.15.40:0/3329891275 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 msgr2=0x7fba180742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba16d9d700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba16d9d700 1 -- 172.21.15.40:0/3329891275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba10009d00 con 0x7fba18101670 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba1659c700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba16d9d700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18072e40 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fba1000b070 tx=0x7fba1000bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba1000e040 con 0x7fba18101670 2023-07-14T11:11:37.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba1819fc40 con 0x7fba18101670 2023-07-14T11:11:37.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fba10009480 con 0x7fba18101670 2023-07-14T11:11:37.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fba100052a0 con 0x7fba18101670 2023-07-14T11:11:37.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.557+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba181a0060 con 0x7fba18101670 2023-07-14T11:11:37.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.558+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba18106980 con 0x7fba18101670 2023-07-14T11:11:37.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.560+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fba1001d480 con 0x7fba18101670 2023-07-14T11:11:37.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.561+0000 7fba077fe700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 0x7fba000a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:37.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.562+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fba100cdf00 con 0x7fba18101670 2023-07-14T11:11:37.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.562+0000 7fba1659c700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 0x7fba000a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:37.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.562+0000 7fba1659c700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 0x7fba000a3600 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fba08005950 tx=0x7fba08009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:37.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.565+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fba10097330 con 0x7fba18101670 2023-07-14T11:11:37.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.774+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7fba18061120 con 0x7fba18101670 2023-07-14T11:11:37.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.775+0000 7fba077fe700 1 -- 172.21.15.40:0/3329891275 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7fba1009a950 con 0x7fba18101670 2023-07-14T11:11:37.777 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 msgr2=0x7fba000a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 0x7fba000a3600 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7fba08005950 tx=0x7fba08009260 comp rx=0 tx=0).stop 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 msgr2=0x7fba18072e40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18072e40 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fba1000b070 tx=0x7fba1000bc80 comp rx=0 tx=0).stop 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 shutdown_connections 2023-07-14T11:11:37.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fba000a1150 0x7fba000a3600 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18101670 0x7fba18072e40 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fba180ffaf0 0x7fba180742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 --2- 172.21.15.40:0/3329891275 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fba18100ce0 0x7fba18072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 >> 172.21.15.40:0/3329891275 conn(0x7fba180fb130 msgr2=0x7fba180fd2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 shutdown_connections 2023-07-14T11:11:37.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.777+0000 7fba1cdff700 1 -- 172.21.15.40:0/3329891275 wait complete. 2023-07-14T11:11:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:37 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1409474040' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:37 smithi130 ceph-mon[129405]: pgmap v337: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:37 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3566910524' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:37 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1819614121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:37 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1409474040' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:37 smithi040 ceph-mon[97954]: pgmap v337: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:37 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3566910524' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:37 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1819614121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.798 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:37.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:37 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1409474040' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:37 smithi045 ceph-mon[128791]: pgmap v337: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:11:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:37 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3566910524' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:37 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1819614121' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:38.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 -- 172.21.15.40:0/410541133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 msgr2=0x7f8558103340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 --2- 172.21.15.40:0/410541133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558103340 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f854c009990 tx=0x7f854c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 -- 172.21.15.40:0/410541133 shutdown_connections 2023-07-14T11:11:38.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 --2- 172.21.15.40:0/410541133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558103880 0x7f855810a140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 --2- 172.21.15.40:0/410541133 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558101d00 0x7f8558102110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 --2- 172.21.15.40:0/410541133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558103340 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 -- 172.21.15.40:0/410541133 >> 172.21.15.40:0/410541133 conn(0x7f85580fd320 msgr2=0x7f85580ff750 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:38.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 -- 172.21.15.40:0/410541133 shutdown_connections 2023-07-14T11:11:38.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:37.999+0000 7f855d5ff700 1 -- 172.21.15.40:0/410541133 wait complete. 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.000+0000 7f855d5ff700 1 Processor -- start 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.000+0000 7f855d5ff700 1 -- start start 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 0x7f8558197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558103880 0x7f855819efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f855819f590 con 0x7f8558102ef0 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f855819f6d0 con 0x7f8558103880 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f855d5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f855819f860 con 0x7f8558101d00 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39622/0 (socket says 172.21.15.40:39622) 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 -- 172.21.15.40:0/3882571031 learned_addr learned my addr 172.21.15.40:0/3882571031 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558103880 0x7f855819efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f8556ffd700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 0x7f8558197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 -- 172.21.15.40:0/3882571031 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 msgr2=0x7f8558197a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 0x7f8558197a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 -- 172.21.15.40:0/3882571031 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558103880 msgr2=0x7f855819efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558103880 0x7f855819efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f85567fc700 1 -- 172.21.15.40:0/3882571031 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f854c0095f0 con 0x7f8558102ef0 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.001+0000 7f8556ffd700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 0x7f8558197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f85567fc700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f854c00b620 tx=0x7f854c005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f854c01d070 con 0x7f8558102ef0 2023-07-14T11:11:38.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f854c022470 con 0x7f8558102ef0 2023-07-14T11:11:38.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f855d5ff700 1 -- 172.21.15.40:0/3882571031 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f855819fae0 con 0x7f8558102ef0 2023-07-14T11:11:38.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f854c017650 con 0x7f8558102ef0 2023-07-14T11:11:38.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.002+0000 7f855d5ff700 1 -- 172.21.15.40:0/3882571031 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f855819ff60 con 0x7f8558102ef0 2023-07-14T11:11:38.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.003+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f85380052f0 con 0x7f8558102ef0 2023-07-14T11:11:38.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.005+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f854c0225e0 con 0x7f8558102ef0 2023-07-14T11:11:38.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.006+0000 7f853ffff700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 0x7f85400a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.006+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f854c0ce140 con 0x7f8558102ef0 2023-07-14T11:11:38.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.007+0000 7f8556ffd700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 0x7f85400a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.007+0000 7f8556ffd700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 0x7f85400a36d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f8558102d50 tx=0x7f8548009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.010+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f854c097570 con 0x7f8558102ef0 2023-07-14T11:11:38.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.264+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8538005f40 con 0x7f8558102ef0 2023-07-14T11:11:38.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.265+0000 7f853ffff700 1 -- 172.21.15.40:0/3882571031 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f854c09ab90 con 0x7f8558102ef0 2023-07-14T11:11:38.266 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:38.267 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:38.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 msgr2=0x7f85400a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 0x7f85400a36d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f8558102d50 tx=0x7f8548009260 comp rx=0 tx=0).stop 2023-07-14T11:11:38.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 msgr2=0x7f8558197f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f854c00b620 tx=0x7f854c005740 comp rx=0 tx=0).stop 2023-07-14T11:11:38.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 shutdown_connections 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f85400a1220 0x7f85400a36d0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8558103880 0x7f855819efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8558101d00 0x7f8558197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 --2- 172.21.15.40:0/3882571031 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8558102ef0 0x7f8558197f50 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 >> 172.21.15.40:0/3882571031 conn(0x7f85580fd320 msgr2=0x7f85581063b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 shutdown_connections 2023-07-14T11:11:38.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.267+0000 7f853dffb700 1 -- 172.21.15.40:0/3882571031 wait complete. 2023-07-14T11:11:38.271 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:38.288 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:38.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 -- 172.21.15.40:0/2328309112 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588100cf0 msgr2=0x7f4588101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 --2- 172.21.15.40:0/2328309112 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588100cf0 0x7f4588101140 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f457c009990 tx=0x7f457c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 -- 172.21.15.40:0/2328309112 shutdown_connections 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 --2- 172.21.15.40:0/2328309112 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4588101680 0x7f4588107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 --2- 172.21.15.40:0/2328309112 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45880ffb00 0x7f45880fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 --2- 172.21.15.40:0/2328309112 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588100cf0 0x7f4588101140 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 -- 172.21.15.40:0/2328309112 >> 172.21.15.40:0/2328309112 conn(0x7f45880fb0e0 msgr2=0x7f45880fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:38.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 -- 172.21.15.40:0/2328309112 shutdown_connections 2023-07-14T11:11:38.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.490+0000 7f458e9ff700 1 -- 172.21.15.40:0/2328309112 wait complete. 2023-07-14T11:11:38.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.491+0000 7f458e9ff700 1 Processor -- start 2023-07-14T11:11:38.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 -- start start 2023-07-14T11:11:38.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 0x7f4588197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 0x7f4588197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f458819f5c0 con 0x7f4588101680 2023-07-14T11:11:38.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f458819f700 con 0x7f4588100cf0 2023-07-14T11:11:38.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458e9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f458819f890 con 0x7f45880ffb00 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458cf9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f45877fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 0x7f4588197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f4587fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 0x7f4588197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f45877fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 0x7f4588197f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54706/0 (socket says 172.21.15.40:54706) 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f45877fe700 1 -- 172.21.15.40:0/400158857 learned_addr learned my addr 172.21.15.40:0/400158857 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.492+0000 7f458cf9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39638/0 (socket says 172.21.15.40:39638) 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 -- 172.21.15.40:0/400158857 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 msgr2=0x7f4588197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 0x7f4588197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 -- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 msgr2=0x7f458819eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 -- 172.21.15.40:0/400158857 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f457c0095f0 con 0x7f45880ffb00 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f458cf9c700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f4587fff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 0x7f4588197a40 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f457800e9c0 tx=0x7f457800ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4578004220 con 0x7f45880ffb00 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4578004380 con 0x7f45880ffb00 2023-07-14T11:11:38.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f458819fb70 con 0x7f45880ffb00 2023-07-14T11:11:38.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4578003be0 con 0x7f45880ffb00 2023-07-14T11:11:38.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.493+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f45881a00d0 con 0x7f45880ffb00 2023-07-14T11:11:38.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.495+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f45881921e0 con 0x7f45880ffb00 2023-07-14T11:11:38.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.497+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4578003d40 con 0x7f45880ffb00 2023-07-14T11:11:38.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.498+0000 7f45857fa700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 0x7f45700a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.498+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4578014070 con 0x7f45880ffb00 2023-07-14T11:11:38.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.498+0000 7f45877fe700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 0x7f45700a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.499+0000 7f45877fe700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 0x7f45700a36d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f457c00b2e0 tx=0x7f457c005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.502+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f45780d2020 con 0x7f45880ffb00 2023-07-14T11:11:38.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:38 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3329891275' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:38.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:38 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3882571031' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:38.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.713+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f458819fd00 con 0x7f45880ffb00 2023-07-14T11:11:38.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.716+0000 7f45857fa700 1 -- 172.21.15.40:0/400158857 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f458819fd00 con 0x7f45880ffb00 2023-07-14T11:11:38.718 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:38.718 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 msgr2=0x7f45700a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 0x7f45700a36d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f457c00b2e0 tx=0x7f457c005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 msgr2=0x7f4588197a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 0x7f4588197a40 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f457800e9c0 tx=0x7f457800ecd0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 shutdown_connections 2023-07-14T11:11:38.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45700a1220 0x7f45700a36d0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4588101680 0x7f458819eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45880ffb00 0x7f4588197a40 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 --2- 172.21.15.40:0/400158857 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4588100cf0 0x7f4588197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 >> 172.21.15.40:0/400158857 conn(0x7f45880fb0e0 msgr2=0x7f45881065b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 shutdown_connections 2023-07-14T11:11:38.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.718+0000 7f458e9ff700 1 -- 172.21.15.40:0/400158857 wait complete. 2023-07-14T11:11:38.739 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:38.739 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:38.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:38 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3329891275' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:38 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3882571031' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:38 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3329891275' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:38 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3882571031' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:38.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 -- 172.21.15.40:0/2984717232 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 msgr2=0x7fd624107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/2984717232 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd624107f20 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7fd61c00b0a0 tx=0x7fd61c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 -- 172.21.15.40:0/2984717232 shutdown_connections 2023-07-14T11:11:38.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/2984717232 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd624107f20 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/2984717232 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd6240ffae0 0x7fd6240ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/2984717232 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd624100cd0 0x7fd624101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 -- 172.21.15.40:0/2984717232 >> 172.21.15.40:0/2984717232 conn(0x7fd6240fb0e0 msgr2=0x7fd6240fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:38.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 -- 172.21.15.40:0/2984717232 shutdown_connections 2023-07-14T11:11:38.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.944+0000 7fd62c1ff700 1 -- 172.21.15.40:0/2984717232 wait complete. 2023-07-14T11:11:38.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 Processor -- start 2023-07-14T11:11:38.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 -- start start 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd624100cd0 0x7fd624195e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd62419cf00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd62419d4d0 con 0x7fd624101660 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd62419d610 con 0x7fd624100cd0 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.946+0000 7fd62c1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd62419d7a0 con 0x7fd6240ffae0 2023-07-14T11:11:38.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47306/0 (socket says 172.21.15.40:47306) 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 -- 172.21.15.40:0/933659749 learned_addr learned my addr 172.21.15.40:0/933659749 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd62a79c700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd62419cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd62979a700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd624100cd0 0x7fd624195e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 -- 172.21.15.40:0/933659749 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd624100cd0 msgr2=0x7fd624195e90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd624100cd0 0x7fd624195e90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 -- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 msgr2=0x7fd62419cf00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:38.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd62419cf00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:38.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 -- 172.21.15.40:0/933659749 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd614009520 con 0x7fd6240ffae0 2023-07-14T11:11:38.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.947+0000 7fd629f9b700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fd62000e970 tx=0x7fd62000ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.948+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd620009790 con 0x7fd6240ffae0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.948+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd62000c7e0 con 0x7fd6240ffae0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.948+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd620003c70 con 0x7fd6240ffae0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.948+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd61c009d00 con 0x7fd6240ffae0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.948+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd62419ddb0 con 0x7fd6240ffae0 2023-07-14T11:11:38.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.949+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd624107000 con 0x7fd6240ffae0 2023-07-14T11:11:38.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.951+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fd62000cc50 con 0x7fd6240ffae0 2023-07-14T11:11:38.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.952+0000 7fd61affd700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 0x7fd6100a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:38.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.952+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fd620014070 con 0x7fd6240ffae0 2023-07-14T11:11:38.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.953+0000 7fd62979a700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 0x7fd6100a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:38.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.953+0000 7fd62979a700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 0x7fd6100a3680 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fd614009df0 tx=0x7fd614009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:38.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:38.954+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fd6200962e0 con 0x7fd6240ffae0 2023-07-14T11:11:39.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.164+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fd62402cac0 con 0x7fd6240ffae0 2023-07-14T11:11:39.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.165+0000 7fd61affd700 1 -- 172.21.15.40:0/933659749 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7fd620099900 con 0x7fd6240ffae0 2023-07-14T11:11:39.166 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 msgr2=0x7fd6100a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 0x7fd6100a3680 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fd614009df0 tx=0x7fd614009260 comp rx=0 tx=0).stop 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 msgr2=0x7fd624195950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fd62000e970 tx=0x7fd62000ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 shutdown_connections 2023-07-14T11:11:39.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd6100a11d0 0x7fd6100a3680 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd624101660 0x7fd62419cf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd6240ffae0 0x7fd624195950 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 --2- 172.21.15.40:0/933659749 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd624100cd0 0x7fd624195e90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 >> 172.21.15.40:0/933659749 conn(0x7fd6240fb0e0 msgr2=0x7fd624104140 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 shutdown_connections 2023-07-14T11:11:39.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.167+0000 7fd62c1ff700 1 -- 172.21.15.40:0/933659749 wait complete. 2023-07-14T11:11:39.187 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:39.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 -- 172.21.15.40:0/3144160561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 msgr2=0x7f4db0107f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3144160561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db0107f30 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f4da8009f90 tx=0x7f4da800b330 comp rx=0 tx=0).stop 2023-07-14T11:11:39.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 -- 172.21.15.40:0/3144160561 shutdown_connections 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3144160561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db0107f30 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3144160561 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 0x7f4db00fff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3144160561 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 -- 172.21.15.40:0/3144160561 >> 172.21.15.40:0/3144160561 conn(0x7f4db00fb130 msgr2=0x7f4db00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 -- 172.21.15.40:0/3144160561 shutdown_connections 2023-07-14T11:11:39.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.394+0000 7f4db4bff700 1 -- 172.21.15.40:0/3144160561 wait complete. 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.395+0000 7f4db4bff700 1 Processor -- start 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 -- start start 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 0x7f4db0197af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db019f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4db019f670 con 0x7f4db0101670 2023-07-14T11:11:39.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4db019f7b0 con 0x7f4db0100ce0 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4db4bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4db019f940 con 0x7f4db00ffaf0 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4da7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4da7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54752/0 (socket says 172.21.15.40:54752) 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4da7fff700 1 -- 172.21.15.40:0/3136411590 learned_addr learned my addr 172.21.15.40:0/3136411590 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4dae59c700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 0x7f4db0197af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.396+0000 7f4daed9d700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db019f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 -- 172.21.15.40:0/3136411590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 msgr2=0x7f4db0197af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 0x7f4db0197af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 -- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 msgr2=0x7f4db019f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db019f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 -- 172.21.15.40:0/3136411590 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d980095f0 con 0x7f4db0100ce0 2023-07-14T11:11:39.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.397+0000 7f4da7fff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f4d9800ec10 tx=0x7f4d9800ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.398+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d9800cc90 con 0x7f4db0100ce0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.398+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4d980044e0 con 0x7f4db0100ce0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.398+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4da8009c40 con 0x7f4db0100ce0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.398+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d98010430 con 0x7f4db0100ce0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.399+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4db019ffa0 con 0x7f4db0100ce0 2023-07-14T11:11:39.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.400+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4db010b320 con 0x7f4db0100ce0 2023-07-14T11:11:39.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.402+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4d98004000 con 0x7f4db0100ce0 2023-07-14T11:11:39.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.403+0000 7f4da77fe700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 0x7f4d9c0a3750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.403+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4d98014070 con 0x7f4db0100ce0 2023-07-14T11:11:39.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.403+0000 7f4dae59c700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 0x7f4d9c0a3750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.404+0000 7f4dae59c700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 0x7f4d9c0a3750 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f4db0100b40 tx=0x7f4da0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:39.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.405+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4d98096b10 con 0x7f4db0100ce0 2023-07-14T11:11:39.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:39 smithi040 ceph-mon[97954]: pgmap v338: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:39 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/400158857' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:39.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:39 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/933659749' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:39.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.616+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f4db01a0670 con 0x7f4db0100ce0 2023-07-14T11:11:39.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.617+0000 7f4da77fe700 1 -- 172.21.15.40:0/3136411590 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7f4d9800f3c0 con 0x7f4db0100ce0 2023-07-14T11:11:39.618 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:39.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 msgr2=0x7f4d9c0a3750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 0x7f4d9c0a3750 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f4db0100b40 tx=0x7f4da0009260 comp rx=0 tx=0).stop 2023-07-14T11:11:39.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 msgr2=0x7f4db0198030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f4d9800ec10 tx=0x7f4d9800ef20 comp rx=0 tx=0).stop 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 shutdown_connections 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4d9c0a12a0 0x7f4d9c0a3750 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4db0101670 0x7f4db019f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4db00ffaf0 0x7f4db0197af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 --2- 172.21.15.40:0/3136411590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4db0100ce0 0x7f4db0198030 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 >> 172.21.15.40:0/3136411590 conn(0x7f4db00fb130 msgr2=0x7f4db00fd290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 shutdown_connections 2023-07-14T11:11:39.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.619+0000 7f4db4bff700 1 -- 172.21.15.40:0/3136411590 wait complete. 2023-07-14T11:11:39.641 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:39.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:39 smithi130 ceph-mon[129405]: pgmap v338: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:39 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/400158857' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:39 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/933659749' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:39 smithi045 ceph-mon[128791]: pgmap v338: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 9 op/s 2023-07-14T11:11:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:39 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/400158857' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:39 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/933659749' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:39.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 -- 172.21.15.40:0/422982781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f000ffae0 msgr2=0x7f3f000ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 --2- 172.21.15.40:0/422982781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f000ffae0 0x7f3f000ffef0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f3efc009990 tx=0x7f3efc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 -- 172.21.15.40:0/422982781 shutdown_connections 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 --2- 172.21.15.40:0/422982781 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f00101660 0x7f3f00107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 --2- 172.21.15.40:0/422982781 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f000ffae0 0x7f3f000ffef0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 --2- 172.21.15.40:0/422982781 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00100cd0 0x7f3f00101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 -- 172.21.15.40:0/422982781 >> 172.21.15.40:0/422982781 conn(0x7f3f000fb0e0 msgr2=0x7f3f000fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 -- 172.21.15.40:0/422982781 shutdown_connections 2023-07-14T11:11:39.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.846+0000 7f3f075ff700 1 -- 172.21.15.40:0/422982781 wait complete. 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.847+0000 7f3f075ff700 1 Processor -- start 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.847+0000 7f3f075ff700 1 -- start start 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 0x7f3f00072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00101660 0x7f3f00072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f000735e0 con 0x7f3f00100cd0 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f00073750 con 0x7f3f000ffae0 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f075ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f0019f8f0 con 0x7f3f00101660 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f04b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 0x7f3f00072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54766/0 (socket says 172.21.15.40:54766) 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 -- 172.21.15.40:0/2119188936 learned_addr learned my addr 172.21.15.40:0/2119188936 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:39.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f05b9c700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00101660 0x7f3f00072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 -- 172.21.15.40:0/2119188936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00101660 msgr2=0x7f3f00072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00101660 0x7f3f00072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 -- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 msgr2=0x7f3f00072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 0x7f3f00072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f0539b700 1 -- 172.21.15.40:0/2119188936 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3efc0095f0 con 0x7f3f000ffae0 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.848+0000 7f3f04b9a700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 0x7f3f00072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3f0539b700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f3efc00b620 tx=0x7f3efc005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:39.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3efc01d070 con 0x7f3f000ffae0 2023-07-14T11:11:39.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f0019fb70 con 0x7f3f000ffae0 2023-07-14T11:11:39.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3efc021410 con 0x7f3f000ffae0 2023-07-14T11:11:39.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3efc017650 con 0x7f3f000ffae0 2023-07-14T11:11:39.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.849+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f0019fff0 con 0x7f3f000ffae0 2023-07-14T11:11:39.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.850+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f00106a60 con 0x7f3f000ffae0 2023-07-14T11:11:39.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.852+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f3efc021580 con 0x7f3f000ffae0 2023-07-14T11:11:39.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.854+0000 7f3ef67fc700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 0x7f3eec0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:39.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.854+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f3efc0ce7a0 con 0x7f3f000ffae0 2023-07-14T11:11:39.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.854+0000 7f3f04b9a700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 0x7f3eec0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:39.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.855+0000 7f3f04b9a700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 0x7f3eec0a3600 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f3ef0011490 tx=0x7f3ef0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:39.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:39.856+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3efc097bd0 con 0x7f3f000ffae0 2023-07-14T11:11:40.067 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.066+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f3f0002ccd0 con 0x7f3f000ffae0 2023-07-14T11:11:40.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.067+0000 7f3ef67fc700 1 -- 172.21.15.40:0/2119188936 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3f0002ccd0 con 0x7f3f000ffae0 2023-07-14T11:11:40.068 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:40.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 msgr2=0x7f3eec0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 0x7f3eec0a3600 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f3ef0011490 tx=0x7f3ef0009260 comp rx=0 tx=0).stop 2023-07-14T11:11:40.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 msgr2=0x7f3f000742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f3efc00b620 tx=0x7f3efc005740 comp rx=0 tx=0).stop 2023-07-14T11:11:40.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 shutdown_connections 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3eec0a1150 0x7f3eec0a3600 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f00101660 0x7f3f00072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f000ffae0 0x7f3f000742c0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 --2- 172.21.15.40:0/2119188936 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f00100cd0 0x7f3f00072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 >> 172.21.15.40:0/2119188936 conn(0x7f3f000fb0e0 msgr2=0x7f3f000fd390 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 shutdown_connections 2023-07-14T11:11:40.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.069+0000 7f3f075ff700 1 -- 172.21.15.40:0/2119188936 wait complete. 2023-07-14T11:11:40.089 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:40.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 -- 172.21.15.40:0/1661807534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c1079a0 msgr2=0x7f2f7c109e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 --2- 172.21.15.40:0/1661807534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c109e20 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7f2f7400b0a0 tx=0x7f2f7400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 -- 172.21.15.40:0/1661807534 shutdown_connections 2023-07-14T11:11:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 --2- 172.21.15.40:0/1661807534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c109e20 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 --2- 172.21.15.40:0/1661807534 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 --2- 172.21.15.40:0/1661807534 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c072f70 0x7f2f7c0733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 -- 172.21.15.40:0/1661807534 >> 172.21.15.40:0/1661807534 conn(0x7f2f7c0fb0e0 msgr2=0x7f2f7c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 -- 172.21.15.40:0/1661807534 shutdown_connections 2023-07-14T11:11:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.295+0000 7f2f82fff700 1 -- 172.21.15.40:0/1661807534 wait complete. 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 Processor -- start 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 -- start start 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c1980d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c19f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7c19f710 con 0x7f2f7c072f70 2023-07-14T11:11:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7c19f850 con 0x7f2f7c1079a0 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.296+0000 7f2f82fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f7c19f9e0 con 0x7f2f7c074550 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39716/0 (socket says 172.21.15.40:39716) 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 -- 172.21.15.40:0/3480151231 learned_addr learned my addr 172.21.15.40:0/3480151231 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f8159c700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c19f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f7bfff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c1980d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 -- 172.21.15.40:0/3480151231 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 msgr2=0x7f2f7c1980d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c1980d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 -- 172.21.15.40:0/3480151231 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 msgr2=0x7f2f7c19f140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c19f140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 -- 172.21.15.40:0/3480151231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f70009520 con 0x7f2f7c072f70 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f8159c700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c19f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f7bfff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c1980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.297+0000 7f2f80d9b700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f2f7000ea30 tx=0x7f2f7000ed40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.298+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f70009820 con 0x7f2f7c072f70 2023-07-14T11:11:40.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.298+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f74009d00 con 0x7f2f7c072f70 2023-07-14T11:11:40.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.298+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2f700044e0 con 0x7f2f7c072f70 2023-07-14T11:11:40.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.298+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2f70003dc0 con 0x7f2f7c072f70 2023-07-14T11:11:40.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.298+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f7c1a0040 con 0x7f2f7c072f70 2023-07-14T11:11:40.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.299+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f7c106050 con 0x7f2f7c072f70 2023-07-14T11:11:40.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.301+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f2f70004000 con 0x7f2f7c072f70 2023-07-14T11:11:40.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.302+0000 7f2f79ffb700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 0x7f2f6c0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.302+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f2f70014070 con 0x7f2f7c072f70 2023-07-14T11:11:40.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.302+0000 7f2f7bfff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 0x7f2f6c0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.303+0000 7f2f7bfff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 0x7f2f6c0a3600 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f2f7c074270 tx=0x7f2f68009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:40.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.306+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f2f70096420 con 0x7f2f7c072f70 2023-07-14T11:11:40.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.515+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f2f7c061230 con 0x7f2f7c072f70 2023-07-14T11:11:40.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.516+0000 7f2f79ffb700 1 -- 172.21.15.40:0/3480151231 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7f2f70016330 con 0x7f2f7c072f70 2023-07-14T11:11:40.518 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 msgr2=0x7f2f6c0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 0x7f2f6c0a3600 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f2f7c074270 tx=0x7f2f68009260 comp rx=0 tx=0).stop 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 msgr2=0x7f2f7c197b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f2f7000ea30 tx=0x7f2f7000ed40 comp rx=0 tx=0).stop 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 shutdown_connections 2023-07-14T11:11:40.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f2f6c0a1150 0x7f2f6c0a3600 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2f7c1079a0 0x7f2f7c19f140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2f7c072f70 0x7f2f7c197b90 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 --2- 172.21.15.40:0/3480151231 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2f7c074550 0x7f2f7c1980d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 >> 172.21.15.40:0/3480151231 conn(0x7f2f7c0fb0e0 msgr2=0x7f2f7c0fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 shutdown_connections 2023-07-14T11:11:40.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.518+0000 7f2f82fff700 1 -- 172.21.15.40:0/3480151231 wait complete. 2023-07-14T11:11:40.539 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 -- 172.21.15.40:0/2009778164 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 msgr2=0x7f44ac107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 --2- 172.21.15.40:0/2009778164 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac107f40 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f44a400b0a0 tx=0x7f44a400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 -- 172.21.15.40:0/2009778164 shutdown_connections 2023-07-14T11:11:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 --2- 172.21.15.40:0/2009778164 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac107f40 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 --2- 172.21.15.40:0/2009778164 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 --2- 172.21.15.40:0/2009778164 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 0x7f44ac101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 -- 172.21.15.40:0/2009778164 >> 172.21.15.40:0/2009778164 conn(0x7f44ac0fb0e0 msgr2=0x7f44ac0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:40.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 -- 172.21.15.40:0/2009778164 shutdown_connections 2023-07-14T11:11:40.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.741+0000 7f44b19ff700 1 -- 172.21.15.40:0/2009778164 wait complete. 2023-07-14T11:11:40.745 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.742+0000 7f44b19ff700 1 Processor -- start 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.742+0000 7f44b19ff700 1 -- start start 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.742+0000 7f44b19ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44b19ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 0x7f44ac198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44b19ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac19f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44b19ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44ac19f6c0 con 0x7f44ac101680 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44b19ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44ac19f800 con 0x7f44ac0ffb00 2023-07-14T11:11:40.746 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44b19ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44ac19f990 con 0x7f44ac100cf0 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac19f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44aaffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44aaffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54786/0 (socket says 172.21.15.40:54786) 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44aaffd700 1 -- 172.21.15.40:0/1896112889 learned_addr learned my addr 172.21.15.40:0/1896112889 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44a3fff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 0x7f44ac198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 -- 172.21.15.40:0/1896112889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 msgr2=0x7f44ac198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 0x7f44ac198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 -- 172.21.15.40:0/1896112889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 msgr2=0x7f44ac197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44ab7fe700 1 -- 172.21.15.40:0/1896112889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44a4009d00 con 0x7f44ac101680 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.743+0000 7f44aaffd700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44ab7fe700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac19f0f0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f44a400b070 tx=0x7f44a400bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a400e040 con 0x7f44ac101680 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44ac19fc10 con 0x7f44ac101680 2023-07-14T11:11:40.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44a4003e10 con 0x7f44ac101680 2023-07-14T11:11:40.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44a4005280 con 0x7f44ac101680 2023-07-14T11:11:40.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.744+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44ac1a0030 con 0x7f44ac101680 2023-07-14T11:11:40.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.745+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f44ac10b330 con 0x7f44ac101680 2023-07-14T11:11:40.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.747+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f44a40053e0 con 0x7f44ac101680 2023-07-14T11:11:40.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.749+0000 7f44b09fd700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 0x7f44980a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:40.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.749+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f44a40ce0a0 con 0x7f44ac101680 2023-07-14T11:11:40.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.749+0000 7f44aaffd700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 0x7f44980a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:40.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.750+0000 7f44aaffd700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 0x7f44980a3650 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f449c005950 tx=0x7f449c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:40.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:40.752+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f44a4097530 con 0x7f44ac101680 2023-07-14T11:11:40.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:40 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3136411590' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:40.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:40 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2119188936' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:40.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:40 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3136411590' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:40.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:40 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2119188936' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:40.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:40 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3136411590' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:40 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2119188936' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:41.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.006+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f44ac1a05e0 con 0x7f44ac101680 2023-07-14T11:11:41.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.007+0000 7f44b09fd700 1 -- 172.21.15.40:0/1896112889 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f44a40aee60 con 0x7f44ac101680 2023-07-14T11:11:41.009 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:41.009 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:41.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.009+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 msgr2=0x7f44980a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.009+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 0x7f44980a3650 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f449c005950 tx=0x7f449c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:41.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.009+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 msgr2=0x7f44ac19f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.009+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac19f0f0 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f44a400b070 tx=0x7f44a400bc80 comp rx=0 tx=0).stop 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 shutdown_connections 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f44980a11a0 0x7f44980a3650 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44ac101680 0x7f44ac19f0f0 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44ac0ffb00 0x7f44ac197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 --2- 172.21.15.40:0/1896112889 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44ac100cf0 0x7f44ac198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 >> 172.21.15.40:0/1896112889 conn(0x7f44ac0fb0e0 msgr2=0x7f44ac1064d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 shutdown_connections 2023-07-14T11:11:41.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.010+0000 7f44b19ff700 1 -- 172.21.15.40:0/1896112889 wait complete. 2023-07-14T11:11:41.013 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:41.030 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:41.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 -- 172.21.15.40:0/2538859015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9554101680 msgr2=0x7f9554107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 --2- 172.21.15.40:0/2538859015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9554101680 0x7f9554107f40 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f954c00b0a0 tx=0x7f954c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 -- 172.21.15.40:0/2538859015 shutdown_connections 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 --2- 172.21.15.40:0/2538859015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9554101680 0x7f9554107f40 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 --2- 172.21.15.40:0/2538859015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f95540ffb00 0x7f95540fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 --2- 172.21.15.40:0/2538859015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 -- 172.21.15.40:0/2538859015 >> 172.21.15.40:0/2538859015 conn(0x7f95540fb0e0 msgr2=0x7f95540fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 -- 172.21.15.40:0/2538859015 shutdown_connections 2023-07-14T11:11:41.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.231+0000 7f95589ff700 1 -- 172.21.15.40:0/2538859015 wait complete. 2023-07-14T11:11:41.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.232+0000 7f95589ff700 1 Processor -- start 2023-07-14T11:11:41.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 -- start start 2023-07-14T11:11:41.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554195e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 0x7f955419cef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f955419d4c0 con 0x7f95540ffb00 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f955419d600 con 0x7f9554101680 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f95589ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f955419d790 con 0x7f9554100cf0 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f955259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f955259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39758/0 (socket says 172.21.15.40:39758) 2023-07-14T11:11:41.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f955259c700 1 -- 172.21.15.40:0/3014911733 learned_addr learned my addr 172.21.15.40:0/3014911733 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f9552d9d700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 0x7f955419cef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.233+0000 7f9551d9b700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554195e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 -- 172.21.15.40:0/3014911733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 msgr2=0x7f9554195e80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554195e80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 -- 172.21.15.40:0/3014911733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 msgr2=0x7f955419cef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 0x7f955419cef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 -- 172.21.15.40:0/3014911733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f954c009d00 con 0x7f95540ffb00 2023-07-14T11:11:41.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f9551d9b700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554195e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f9552d9d700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 0x7f955419cef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f955259c700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f954400e970 tx=0x7f954400ed30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9544009790 con 0x7f95540ffb00 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f95440044e0 con 0x7f95540ffb00 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f955419da70 con 0x7f95540ffb00 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9544003e30 con 0x7f95540ffb00 2023-07-14T11:11:41.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.234+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f955419dfd0 con 0x7f95540ffb00 2023-07-14T11:11:41.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.235+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9554106f90 con 0x7f95540ffb00 2023-07-14T11:11:41.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.237+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9544003f90 con 0x7f95540ffb00 2023-07-14T11:11:41.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.239+0000 7f95437fe700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 0x7f953c0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.239+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f9544014070 con 0x7f95540ffb00 2023-07-14T11:11:41.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.239+0000 7f9551d9b700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 0x7f953c0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.239+0000 7f9551d9b700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 0x7f953c0a36d0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f9548009cb0 tx=0x7f9548009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.242+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f95440962f0 con 0x7f95540ffb00 2023-07-14T11:11:41.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.451+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f955402c850 con 0x7f95540ffb00 2023-07-14T11:11:41.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.452+0000 7f95437fe700 1 -- 172.21.15.40:0/3014911733 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f9544099910 con 0x7f95540ffb00 2023-07-14T11:11:41.455 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:41.455 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 msgr2=0x7f953c0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 0x7f953c0a36d0 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f9548009cb0 tx=0x7f9548009260 comp rx=0 tx=0).stop 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 msgr2=0x7f9554195940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f954400e970 tx=0x7f954400ed30 comp rx=0 tx=0).stop 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 shutdown_connections 2023-07-14T11:11:41.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f953c0a1220 0x7f953c0a36d0 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9554101680 0x7f955419cef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95540ffb00 0x7f9554195940 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 --2- 172.21.15.40:0/3014911733 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9554100cf0 0x7f9554195e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 >> 172.21.15.40:0/3014911733 conn(0x7f95540fb0e0 msgr2=0x7f95541040d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 shutdown_connections 2023-07-14T11:11:41.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.455+0000 7f95589ff700 1 -- 172.21.15.40:0/3014911733 wait complete. 2023-07-14T11:11:41.476 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 27, 'flags': 50, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': True, 'refuse_client_session': False, 'refuse_standby_for_another_fs': False}, 'ever_allowed_features': 32, 'explicitly_allowed_features': 32, 'created': '2023-07-14T11:09:43.127642+0000', 'modified': '2023-07-14T11:10:17.374269+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 300, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'max_xattr_size': 65536, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 5, 'in': [0, 1, 2, 3, 4], 'up': {'mds_0': 24569, 'mds_1': 24565, 'mds_2': 24523, 'mds_3': 24521, 'mds_4': 24511}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24466': {'gid': 24466, 'name': 'l', 'rank': 4, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6833/2180852091', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6832', 'nonce': 2180852091}, {'type': 'v1', 'addr': '172.21.15.130:6833', 'nonce': 2180852091}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24482': {'gid': 24482, 'name': 'd', 'rank': 3, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.40:6837/625677299', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6836', 'nonce': 625677299}, {'type': 'v1', 'addr': '172.21.15.40:6837', 'nonce': 625677299}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24494': {'gid': 24494, 'name': 'e', 'rank': 2, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.45:6835/1717726567', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6834', 'nonce': 1717726567}, {'type': 'v1', 'addr': '172.21.15.45:6835', 'nonce': 1717726567}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24511': {'gid': 24511, 'name': 'f', 'rank': 4, 'incarnation': 25, 'state': 'up:active', 'state_seq': 15, 'addr': '172.21.15.130:6835/2228846875', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6834', 'nonce': 2228846875}, {'type': 'v1', 'addr': '172.21.15.130:6835', 'nonce': 2228846875}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24521': {'gid': 24521, 'name': 'h', 'rank': 3, 'incarnation': 23, 'state': 'up:active', 'state_seq': 14, 'addr': '172.21.15.45:6837/3949707739', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6836', 'nonce': 3949707739}, {'type': 'v1', 'addr': '172.21.15.45:6837', 'nonce': 3949707739}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24523': {'gid': 24523, 'name': 'c', 'rank': 2, 'incarnation': 21, 'state': 'up:active', 'state_seq': 14, 'addr': '172.21.15.130:6837/4266211241', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6836', 'nonce': 4266211241}, {'type': 'v1', 'addr': '172.21.15.130:6837', 'nonce': 4266211241}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24541': {'gid': 24541, 'name': 'i', 'rank': 1, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.130:6839/3942717247', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.130:6838', 'nonce': 3942717247}, {'type': 'v1', 'addr': '172.21.15.130:6839', 'nonce': 3942717247}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24553': {'gid': 24553, 'name': 'g', 'rank': 0, 'incarnation': 0, 'state': 'up:standby-replay', 'state_seq': 1, 'addr': '172.21.15.40:6839/3591632989', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6838', 'nonce': 3591632989}, {'type': 'v1', 'addr': '172.21.15.40:6839', 'nonce': 3591632989}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24565': {'gid': 24565, 'name': 'a', 'rank': 1, 'incarnation': 19, 'state': 'up:active', 'state_seq': 11, 'addr': '172.21.15.40:6841/1989097261', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.40:6840', 'nonce': 1989097261}, {'type': 'v1', 'addr': '172.21.15.40:6841', 'nonce': 1989097261}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}, 'gid_24569': {'gid': 24569, 'name': 'b', 'rank': 0, 'incarnation': 14, 'state': 'up:active', 'state_seq': 3, 'addr': '172.21.15.45:6839/180621035', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.45:6838', 'nonce': 180621035}, {'type': 'v1', 'addr': '172.21.15.45:6839', 'nonce': 180621035}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138322906710015, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3, 4], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'bal_rank_mask': '-1', 'standby_count_wanted': 0} 2023-07-14T11:11:41.476 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 5/5 2023-07-14T11:11:41.477 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.f status 2023-07-14T11:11:41.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 -- 172.21.15.40:0/102121538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c100ffae0 msgr2=0x7f5c100ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 --2- 172.21.15.40:0/102121538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c100ffae0 0x7f5c100ffef0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c009990 tx=0x7f5c0c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 -- 172.21.15.40:0/102121538 shutdown_connections 2023-07-14T11:11:41.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 --2- 172.21.15.40:0/102121538 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c10101660 0x7f5c10107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 --2- 172.21.15.40:0/102121538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c100ffae0 0x7f5c100ffef0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 --2- 172.21.15.40:0/102121538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 -- 172.21.15.40:0/102121538 >> 172.21.15.40:0/102121538 conn(0x7f5c100fb0e0 msgr2=0x7f5c100fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 -- 172.21.15.40:0/102121538 shutdown_connections 2023-07-14T11:11:41.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.680+0000 7f5c183ff700 1 -- 172.21.15.40:0/102121538 wait complete. 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.681+0000 7f5c183ff700 1 Processor -- start 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.681+0000 7f5c183ff700 1 -- start start 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c10101660 0x7f5c10072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c100735e0 con 0x7f5c10101660 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c10073750 con 0x7f5c100ffae0 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c183ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c1019f8c0 con 0x7f5c10100cd0 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54818/0 (socket says 172.21.15.40:54818) 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 -- 172.21.15.40:0/1767581838 learned_addr learned my addr 172.21.15.40:0/1767581838 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1699c700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c10101660 0x7f5c10072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1599a700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 -- 172.21.15.40:0/1767581838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 msgr2=0x7f5c10072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 -- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c10101660 msgr2=0x7f5c10072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c10101660 0x7f5c10072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1619b700 1 -- 172.21.15.40:0/1767581838 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c0c0095f0 con 0x7f5c100ffae0 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.682+0000 7f5c1599a700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c1619b700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c005f50 tx=0x7f5c0c005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c077fe700 1 -- 172.21.15.40:0/1767581838 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0c01d070 con 0x7f5c100ffae0 2023-07-14T11:11:41.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c077fe700 1 -- 172.21.15.40:0/1767581838 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c0c022470 con 0x7f5c100ffae0 2023-07-14T11:11:41.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c1019fb40 con 0x7f5c100ffae0 2023-07-14T11:11:41.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c077fe700 1 -- 172.21.15.40:0/1767581838 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0c017650 con 0x7f5c100ffae0 2023-07-14T11:11:41.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.683+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c1019ffc0 con 0x7f5c100ffae0 2023-07-14T11:11:41.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.686+0000 7f5c077fe700 1 -- 172.21.15.40:0/1767581838 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f5c0c022870 con 0x7f5c100ffae0 2023-07-14T11:11:41.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.687+0000 7f5c2bd43b80 1 Processor -- start 2023-07-14T11:11:41.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 -- start start 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488f480 0x5576a488f890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a4890690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a4794c60 con 0x5576a4890c20 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a47e79f0 con 0x5576a488fdd0 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a47e0e80 con 0x5576a488f480 2023-07-14T11:11:41.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c077fe700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 0x7f5bfc0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c1599a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c077fe700 1 -- 172.21.15.40:0/1767581838 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5c0c0ce230 con 0x7f5c100ffae0 2023-07-14T11:11:41.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c1599a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39784/0 (socket says 172.21.15.40:39784) 2023-07-14T11:11:41.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c1599a700 1 -- 172.21.15.40:0/4214924863 learned_addr learned my addr 172.21.15.40:0/4214924863 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c1699c700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 0x7f5bfc0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.688+0000 7f5c1619b700 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a4890690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 -- 172.21.15.40:0/4214924863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488f480 msgr2=0x5576a488f890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488f480 0x5576a488f890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 -- 172.21.15.40:0/4214924863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 msgr2=0x5576a4890690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a4890690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 -- 172.21.15.40:0/4214924863 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5576a460ed90 con 0x5576a4890c20 2023-07-14T11:11:41.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1599a700 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f5c0000bd90 tx=0x7f5c0000f640 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ef5ec7c631a3d8e7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c1699c700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 0x7f5bfc0a3680 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f5c10074920 tx=0x7f5c08009920 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c04bfe700 1 -- 172.21.15.40:0/4214924863 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0000fda0 con 0x5576a4890c20 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c04bfe700 1 -- 172.21.15.40:0/4214924863 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c00011900 con 0x5576a4890c20 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.689+0000 7f5c04bfe700 1 -- 172.21.15.40:0/4214924863 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c000106a0 con 0x5576a4890c20 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/4214924863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 msgr2=0x5576a4891090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f5c0000bd90 tx=0x7f5c0000f640 comp rx=0 tx=0).stop 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/4214924863 shutdown_connections 2023-07-14T11:11:41.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a4890c20 0x5576a4891090 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488f480 0x5576a488f890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/4214924863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a4890690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/4214924863 >> 172.21.15.40:0/4214924863 conn(0x5576a4884d50 msgr2=0x5576a488a180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/4214924863 shutdown_connections 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.690+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/4214924863 wait complete. 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 -- start start 2023-07-14T11:11:41.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 Processor -- start 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488f480 0x5576a48990a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 0x5576a48995e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48a0620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a4780a70 con 0x5576a488fdd0 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a4780770 con 0x5576a488f480 2023-07-14T11:11:41.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a46648d0 con 0x5576a4890c20 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.691+0000 7f5c2bd43b80 10 client.0 _ll_register_callbacks cb 0x5576a477e710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1699c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48a0620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 0x5576a48995e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1699c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48a0620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47444/0 (socket says 172.21.15.40:47444) 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1699c700 1 -- 172.21.15.40:0/3867384944 learned_addr learned my addr 172.21.15.40:0/3867384944 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1619b700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488f480 0x5576a48990a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 -- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 msgr2=0x5576a48a0620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48a0620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 -- 172.21.15.40:0/3867384944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488f480 msgr2=0x5576a48990a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488f480 0x5576a48990a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 -- 172.21.15.40:0/3867384944 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c0000b9c0 con 0x5576a488fdd0 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5c1599a700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 0x5576a48995e0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f5c0000f990 tx=0x7f5c0000fdd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9d5def408e97d8c1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5bf9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.692+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c00020070 con 0x5576a488fdd0 2023-07-14T11:11:41.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5c00011bb0 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5576a473a4a0 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5c00011d70 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 10 client.15312 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5576a45f7c10 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5576a473a4a0 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f5c000047e0 con 0x5576a488fdd0 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 10 client.15312 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:41.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 10 client.15312 resolve_mds: resolved name 'f' to daemon mds.f 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 20 client.15312 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48a41b0 0x5576a48a3660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 4 client.15312 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.693+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5576a47d9e60 con 0x5576a48a41b0 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.694+0000 7f5c1619b700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48a41b0 0x5576a48a3660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.694+0000 7f5bf9ffb700 10 client.15312 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.694+0000 7f5c1619b700 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48a41b0 0x5576a48a3660 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=19608511b9ff08e3 server_cookie=dba837cf933303ee in_seq=0 out_seq=0 2023-07-14T11:11:41.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.695+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/3867384944 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5576a47d9e60 con 0x5576a48a41b0 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.695+0000 7f5bf9ffb700 10 client.15312 handle_command_reply: tid=0 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.695+0000 7f5c2bd43b80 1 client.15312 shutdown 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 msgr2=0x5576a48995e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 0x5576a48995e0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f5c0000f990 tx=0x7f5c0000fdd0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 shutdown_connections 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48a41b0 0x5576a48a3660 crc :-1 s=READY pgs=20 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48a0620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488f480 0x5576a48990a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5bf9ffb700 0 client.15312 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/3867384944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488fdd0 0x5576a48995e0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 >> 172.21.15.40:0/3867384944 conn(0x5576a4884d50 msgr2=0x5576a4889860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 shutdown_connections 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/3867384944 wait complete. 2023-07-14T11:11:41.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.696+0000 7f5c2bd43b80 20 client.15312 trim_cache size 0 max 16384 2023-07-14T11:11:41.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.714+0000 7f5c2bd43b80 1 Processor -- start 2023-07-14T11:11:41.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 -- start start 2023-07-14T11:11:41.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a48a0540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48acf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a473a4a0 con 0x5576a488f480 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a489a290 con 0x5576a488fdd0 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a4670de0 con 0x5576a4890c20 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39788/0 (socket says 172.21.15.40:39788) 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 -- 172.21.15.40:0/2474180101 learned_addr learned my addr 172.21.15.40:0/2474180101 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1599a700 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a48a0540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1699c700 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48acf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 -- 172.21.15.40:0/2474180101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 msgr2=0x5576a48acf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48acf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 -- 172.21.15.40:0/2474180101 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 msgr2=0x5576a48a0540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a48a0540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.715+0000 7f5c1619b700 1 -- 172.21.15.40:0/2474180101 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5576a4635200 con 0x5576a488f480 2023-07-14T11:11:41.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c006010 tx=0x7f5c0c0cb850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6eb730733ec0940d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c053ff700 1 -- 172.21.15.40:0/2474180101 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0c09f120 con 0x5576a488f480 2023-07-14T11:11:41.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c053ff700 1 -- 172.21.15.40:0/2474180101 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c0c09b320 con 0x5576a488f480 2023-07-14T11:11:41.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c053ff700 1 -- 172.21.15.40:0/2474180101 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0c099750 con 0x5576a488f480 2023-07-14T11:11:41.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2474180101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 msgr2=0x5576a4899d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.716+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c006010 tx=0x7f5c0c0cb850 comp rx=0 tx=0).stop 2023-07-14T11:11:41.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2474180101 shutdown_connections 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a4890c20 0x5576a48acf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a4899d20 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2474180101 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a488fdd0 0x5576a48a0540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2474180101 >> 172.21.15.40:0/2474180101 conn(0x5576a4884d50 msgr2=0x5576a48862c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2474180101 shutdown_connections 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2474180101 wait complete. 2023-07-14T11:11:41.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 -- start start 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.717+0000 7f5c2bd43b80 1 Processor -- start 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488fdd0 0x5576a489fac0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a4890c20 0x5576a48a00a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a48a0800 con 0x5576a488f480 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a48a0970 con 0x5576a4890c20 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5576a48f80d0 con 0x5576a488fdd0 2023-07-14T11:11:41.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c2bd43b80 10 client.0 _ll_register_callbacks cb 0x7f5c0c005880 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c1619b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39790/0 (socket says 172.21.15.40:39790) 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.718+0000 7f5c1619b700 1 -- 172.21.15.40:0/2400519257 learned_addr learned my addr 172.21.15.40:0/2400519257 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1599a700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488fdd0 0x5576a489fac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1699c700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a4890c20 0x5576a48a00a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 -- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488fdd0 msgr2=0x5576a489fac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488fdd0 0x5576a489fac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 -- 172.21.15.40:0/2400519257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a4890c20 msgr2=0x5576a48a00a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a4890c20 0x5576a48a00a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 -- 172.21.15.40:0/2400519257 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c1019ffc0 con 0x5576a488f480 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c1619b700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c09b980 tx=0x7f5c0c000bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9360eb34b2623d64 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5bf9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5c0c09a810 con 0x5576a488f480 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5c0c0cbe40 con 0x5576a488f480 2023-07-14T11:11:41.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5c00011900 con 0x5576a488f480 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5c0c09a970 con 0x5576a488f480 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c2bd43b80 10 client.15318 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.719+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5576a460d260 con 0x5576a488f480 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5c00011900 con 0x5576a488f480 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f5c0c00fad0 con 0x5576a488f480 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 10 client.15318 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 10 client.15318 resolve_mds: resolved name 'f' to daemon mds.f 2023-07-14T11:11:41.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 20 client.15318 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48fca50 0x5576a48f93e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 4 client.15318 mds_command: new command op to 24511 tid=0[{"prefix": "status"}] 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5576a479c000 con 0x5576a48fca50 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.720+0000 7f5c1599a700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48fca50 0x5576a48f93e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c1599a700 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48fca50 0x5576a48f93e0 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=ab333c3734a249b2 server_cookie=4214c22bb95ef5c in_seq=0 out_seq=0 2023-07-14T11:11:41.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5bf9ffb700 10 client.15318 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:41.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5bf9ffb700 1 -- 172.21.15.40:0/2400519257 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x5576a479c000 con 0x5576a48fca50 2023-07-14T11:11:41.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5bf9ffb700 10 client.15318 handle_command_reply: tid=0 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 client.15318 shutdown 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 msgr2=0x5576a489d570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c09b980 tx=0x7f5c0c000bc0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 shutdown_connections 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5576a48fca50 0x5576a48f93e0 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5576a4890c20 0x5576a48a00a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5576a488f480 0x5576a489d570 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 --2- 172.21.15.40:0/2400519257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5576a488fdd0 0x5576a489fac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5bf9ffb700 0 client.15318 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 >> 172.21.15.40:0/2400519257 conn(0x5576a4884d50 msgr2=0x5576a48acf20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.721+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 shutdown_connections 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c2bd43b80 1 -- 172.21.15.40:0/2400519257 wait complete. 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c2bd43b80 20 client.15318 trim_cache size 0 max 16384 2023-07-14T11:11:41.731 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 4, 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24511, 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:41.732 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 85.355549088000004, 2023-07-14T11:11:41.733 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:41.733 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:41.733 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:41.733 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 138.82827497299999 2023-07-14T11:11:41.733 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 msgr2=0x7f5bfc0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 0x7f5bfc0a3680 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f5c10074920 tx=0x7f5c08009920 comp rx=0 tx=0).stop 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 msgr2=0x7f5c100742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f5c0c005f50 tx=0x7f5c0c005740 comp rx=0 tx=0).stop 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 shutdown_connections 2023-07-14T11:11:41.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f5bfc0a11d0 0x7f5bfc0a3680 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c10101660 0x7f5c10072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5c100ffae0 0x7f5c100742c0 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 --2- 172.21.15.40:0/1767581838 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5c10100cd0 0x7f5c10072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 >> 172.21.15.40:0/1767581838 conn(0x7f5c100fb0e0 msgr2=0x7f5c100fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 shutdown_connections 2023-07-14T11:11:41.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.722+0000 7f5c183ff700 1 -- 172.21.15.40:0/1767581838 wait complete. 2023-07-14T11:11:41.739 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:41 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3480151231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:41.739 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:41 smithi040 ceph-mon[97954]: pgmap v339: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:41.740 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:41 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1896112889' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:41.741 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.h status 2023-07-14T11:11:41.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:41 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3480151231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:41.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:41 smithi130 ceph-mon[129405]: pgmap v339: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:41 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1896112889' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:41.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:41 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3480151231' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:41 smithi045 ceph-mon[128791]: pgmap v339: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:41 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1896112889' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:41.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/2937547414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 msgr2=0x7f8e04072940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/2937547414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04072940 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f8e00009990 tx=0x7f8e00009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/2937547414 shutdown_connections 2023-07-14T11:11:41.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/2937547414 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e04109db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/2937547414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04072940 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/2937547414 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 0x7f8e04073380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/2937547414 >> 172.21.15.40:0/2937547414 conn(0x7f8e040fb0a0 msgr2=0x7f8e040fd4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/2937547414 shutdown_connections 2023-07-14T11:11:41.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.944+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/2937547414 wait complete. 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.945+0000 7f8e0c1ff700 1 Processor -- start 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.945+0000 7f8e0c1ff700 1 -- start start 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 0x7f8e04197980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0419f500 con 0x7f8e040744f0 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0419f640 con 0x7f8e04107930 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0c1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e0419f7d0 con 0x7f8e04072f10 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54874/0 (socket says 172.21.15.40:54874) 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39816/0 (socket says 172.21.15.40:39816) 2023-07-14T11:11:41.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 -- 172.21.15.40:0/140231591 learned_addr learned my addr 172.21.15.40:0/140231591 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 0x7f8e04197980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 -- 172.21.15.40:0/140231591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 msgr2=0x7f8e04197980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 0x7f8e04197980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 -- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 msgr2=0x7f8e04197ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0a79c700 1 -- 172.21.15.40:0/140231591 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e000095f0 con 0x7f8e04107930 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.946+0000 7f8e0979a700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:41.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8e0a79c700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f8dfc00ef30 tx=0x7f8dfc00c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8dfaffd700 1 -- 172.21.15.40:0/140231591 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dfc018070 con 0x7f8e04107930 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e0419fa50 con 0x7f8e04107930 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8dfaffd700 1 -- 172.21.15.40:0/140231591 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8dfc00f040 con 0x7f8e04107930 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8dfaffd700 1 -- 172.21.15.40:0/140231591 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dfc00cd00 con 0x7f8e04107930 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.947+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e0419ff60 con 0x7f8e04107930 2023-07-14T11:11:41.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.950+0000 7f8dfaffd700 1 -- 172.21.15.40:0/140231591 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f8dfc003a00 con 0x7f8e04107930 2023-07-14T11:11:41.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 Processor -- start 2023-07-14T11:11:41.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 -- start start 2023-07-14T11:11:41.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x5567619f7690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f7c20 0x5567619f8090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567618fbc60 con 0x5567619f7c20 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55676194e9f0 con 0x5567619f6480 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.952+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556761947e80 con 0x5567619f6dd0 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54878/0 (socket says 172.21.15.40:54878) 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 -- 172.21.15.40:0/3861642162 learned_addr learned my addr 172.21.15.40:0/3861642162 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8dfaffd700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 0x7f8df00a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f7c20 0x5567619f8090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x5567619f7690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8dfaffd700 1 -- 172.21.15.40:0/140231591 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f8dfc0cc7c0 con 0x7f8e04107930 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0a79c700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 0x7f8df00a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 -- 172.21.15.40:0/3861642162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 msgr2=0x5567619f7690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x5567619f7690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 -- 172.21.15.40:0/3861642162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f7c20 msgr2=0x5567619f8090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f7c20 0x5567619f8090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 -- 172.21.15.40:0/3861642162 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556761775d90 con 0x5567619f6480 2023-07-14T11:11:41.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.953+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f8df4012c40 tx=0x7f8df400bc30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a68343e4cdb2458a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e0a79c700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 0x7f8df00a3600 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f8dfc009c10 tx=0x7f8dfc000f40 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8deffff700 1 -- 172.21.15.40:0/3861642162 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df4004020 con 0x5567619f6480 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8deffff700 1 -- 172.21.15.40:0/3861642162 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8df4004830 con 0x5567619f6480 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8deffff700 1 -- 172.21.15.40:0/3861642162 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df40053c0 con 0x5567619f6480 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3861642162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 msgr2=0x5567619f6890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f8df4012c40 tx=0x7f8df400bc30 comp rx=0 tx=0).stop 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3861642162 shutdown_connections 2023-07-14T11:11:41.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f7c20 0x5567619f8090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x5567619f6890 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3861642162 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x5567619f7690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.954+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3861642162 >> 172.21.15.40:0/3861642162 conn(0x5567619ebd50 msgr2=0x5567619f1180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.955+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3861642162 shutdown_connections 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.955+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3861642162 wait complete. 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.955+0000 7f8e1fb04b80 1 -- start start 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.955+0000 7f8e1fb04b80 1 Processor -- start 2023-07-14T11:11:41.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a00650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f7c20 0x556761a07690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567618e7a70 con 0x5567619f6480 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567618e7770 con 0x5567619f7c20 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567617cb8d0 con 0x5567619f6dd0 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e1fb04b80 10 client.0 _ll_register_callbacks cb 0x5567618e5710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39836/0 (socket says 172.21.15.40:39836) 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e0979a700 1 -- 172.21.15.40:0/602991926 learned_addr learned my addr 172.21.15.40:0/602991926 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a00650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.956+0000 7f8e0979a700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f7c20 0x556761a07690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 -- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f7c20 msgr2=0x556761a07690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f7c20 0x556761a07690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 -- 172.21.15.40:0/602991926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 msgr2=0x556761a00110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 -- 172.21.15.40:0/602991926 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8df401a040 con 0x5567619f6dd0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e0979a700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a00650 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8e00000c00 tx=0x7f8e00005c00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4d2de24dca1268aa server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8ded7fa700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e0001d030 con 0x5567619f6dd0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5567618a14a0 con 0x5567619f6dd0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.957+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f8e00004700 con 0x5567619f6dd0 2023-07-14T11:11:41.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.958+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8e000048c0 con 0x5567619f6dd0 2023-07-14T11:11:41.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.958+0000 7f8e1fb04b80 10 client.25336 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:41.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.958+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55676175ec10 con 0x5567619f6dd0 2023-07-14T11:11:41.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.958+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5567618a14a0 con 0x5567619f6dd0 2023-07-14T11:11:41.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.958+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8e0000f9c0 con 0x5567619f6dd0 2023-07-14T11:11:41.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 10 client.25336 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:41.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 10 client.25336 resolve_mds: resolved name 'h' to daemon mds.h 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 20 client.25336 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x556761a0b3e0 0x556761a0a790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 4 client.25336 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x556761940e60 con 0x556761a0b3e0 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e0979a700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x556761a0b3e0 0x556761a0a790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8ded7fa700 10 client.25336 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:41.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.959+0000 7f8e0979a700 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x556761a0b3e0 0x556761a0a790 crc :-1 s=READY pgs=25 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=9ea532a264ac38bf server_cookie=ead5e5775645c80 in_seq=0 out_seq=0 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.960+0000 7f8ded7fa700 1 -- 172.21.15.40:0/602991926 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x556761940e60 con 0x556761a0b3e0 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.960+0000 7f8ded7fa700 10 client.25336 handle_command_reply: tid=0 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.960+0000 7f8e1fb04b80 1 client.25336 shutdown 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.960+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 msgr2=0x556761a00650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.960+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a00650 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f8e00000c00 tx=0x7f8e00005c00 comp rx=0 tx=0).stop 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 shutdown_connections 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x556761a0b3e0 0x556761a0a790 crc :-1 s=READY pgs=25 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f7c20 0x556761a07690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619f6480 0x556761a00110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8ded7fa700 0 client.25336 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/602991926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a00650 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 >> 172.21.15.40:0/602991926 conn(0x5567619ebd50 msgr2=0x5567619f0860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 shutdown_connections 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/602991926 wait complete. 2023-07-14T11:11:41.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.961+0000 7f8e1fb04b80 20 client.25336 trim_cache size 0 max 16384 2023-07-14T11:11:41.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 Processor -- start 2023-07-14T11:11:41.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 -- start start 2023-07-14T11:11:41.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x5567619fe010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a0a790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567619167d0 con 0x5567619ebd50 2023-07-14T11:11:41.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567617d7e10 con 0x5567619f6480 2023-07-14T11:11:41.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.979+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5567618e3d50 con 0x5567619f6dd0 2023-07-14T11:11:41.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x5567619fe010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54900/0 (socket says 172.21.15.40:54900) 2023-07-14T11:11:41.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 -- 172.21.15.40:0/2003660199 learned_addr learned my addr 172.21.15.40:0/2003660199 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 -- 172.21.15.40:0/2003660199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 msgr2=0x556761a0a790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a0a790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 -- 172.21.15.40:0/2003660199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 msgr2=0x5567619fe010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x5567619fe010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 -- 172.21.15.40:0/2003660199 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55676179c200 con 0x5567619f6480 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.980+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f8e0000c010 tx=0x7f8e00005f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a5e7ddac9b7810bb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8df8ff9700 1 -- 172.21.15.40:0/2003660199 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e0001d040 con 0x5567619f6480 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8df8ff9700 1 -- 172.21.15.40:0/2003660199 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e0001e040 con 0x5567619f6480 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8df8ff9700 1 -- 172.21.15.40:0/2003660199 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e00024940 con 0x5567619f6480 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/2003660199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 msgr2=0x556761a07650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f8e0000c010 tx=0x7f8e00005f00 comp rx=0 tx=0).stop 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/2003660199 shutdown_connections 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a0a790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x5567619fe010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/2003660199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a07650 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/2003660199 >> 172.21.15.40:0/2003660199 conn(0x5567619ff1c0 msgr2=0x5567619f0860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/2003660199 shutdown_connections 2023-07-14T11:11:41.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.981+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/2003660199 wait complete. 2023-07-14T11:11:41.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 -- start start 2023-07-14T11:11:41.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 Processor -- start 2023-07-14T11:11:41.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x556761a05640 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a5f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556761a5fef0 con 0x5567619ebd50 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x556761a60060 con 0x5567619f6480 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556761a60220 con 0x5567619f6dd0 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.982+0000 7f8e1fb04b80 10 client.0 _ll_register_callbacks cb 0x5567618e5710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x556761a05640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e09f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e09f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54906/0 (socket says 172.21.15.40:54906) 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e09f9b700 1 -- 172.21.15.40:0/3701292768 learned_addr learned my addr 172.21.15.40:0/3701292768 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 -- 172.21.15.40:0/3701292768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 msgr2=0x556761a5f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a5f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 -- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 msgr2=0x556761a5d250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 -- 172.21.15.40:0/3701292768 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55676175ec10 con 0x5567619ebd50 2023-07-14T11:11:41.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8e0979a700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x556761a05640 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x556761a0c5e0 tx=0x7f8df400bae0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=60987bd9f377d1f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.983+0000 7f8ded7fa700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8df4026040 con 0x5567619ebd50 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8df4004830 con 0x5567619ebd50 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f8df402c2c0 con 0x5567619ebd50 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8df402c4a0 con 0x5567619ebd50 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8e1fb04b80 10 client.15336 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:41.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x556761774260 con 0x5567619ebd50 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8df4004830 con 0x5567619ebd50 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.984+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8df402c900 con 0x5567619ebd50 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 10 client.15336 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 10 client.15336 resolve_mds: resolved name 'h' to daemon mds.h 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 20 client.15336 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5567619f7c20 0x556761a61850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:41.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 4 client.15336 mds_command: new command op to 24521 tid=0[{"prefix": "status"}] 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "status"}) v1 -- 0x556761903000 con 0x5567619f7c20 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5567619f7c20 0x556761a61850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8e09f9b700 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5567619f7c20 0x556761a61850 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=f46419cbde26a5ca server_cookie=f5052dcb79f22cca in_seq=0 out_seq=0 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.985+0000 7f8ded7fa700 10 client.15336 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8ded7fa700 1 -- 172.21.15.40:0/3701292768 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+327 (crc 0 0 0) 0x556761903000 con 0x5567619f7c20 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8ded7fa700 10 client.15336 handle_command_reply: tid=0 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 client.15336 shutdown 2023-07-14T11:11:41.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 msgr2=0x556761a05640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x556761a05640 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x556761a0c5e0 tx=0x7f8df400bae0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 shutdown_connections 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x5567619f7c20 0x556761a61850 crc :-1 s=READY pgs=26 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5567619f6dd0 0x556761a5f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5567619ebd50 0x556761a05640 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 --2- 172.21.15.40:0/3701292768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5567619f6480 0x556761a5d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8ded7fa700 0 client.15336 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:11:41.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 >> 172.21.15.40:0/3701292768 conn(0x5567619ff1c0 msgr2=0x556761a0a790 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 shutdown_connections 2023-07-14T11:11:41.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 1 -- 172.21.15.40:0/3701292768 wait complete. 2023-07-14T11:11:41.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.986+0000 7f8e1fb04b80 20 client.15336 trim_cache size 0 max 16384 2023-07-14T11:11:41.995 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:41.995 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 3, 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24521, 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 86.627905088999995, 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:41.996 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:41.997 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:41.997 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 134.684075463 2023-07-14T11:11:41.997 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:41.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 msgr2=0x7f8df00a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 0x7f8df00a3600 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f8dfc009c10 tx=0x7f8dfc000f40 comp rx=0 tx=0).stop 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 msgr2=0x7f8e0419ef30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f8dfc00ef30 tx=0x7f8dfc00c3a0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 shutdown_connections 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8df00a1150 0x7f8df00a3600 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e04107930 0x7f8e0419ef30 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e04072f10 0x7f8e04197980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 --2- 172.21.15.40:0/140231591 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e040744f0 0x7f8e04197ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:41.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 >> 172.21.15.40:0/140231591 conn(0x7f8e040fb0a0 msgr2=0x7f8e040fd480 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:41.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 shutdown_connections 2023-07-14T11:11:41.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:41.987+0000 7f8e0c1ff700 1 -- 172.21.15.40:0/140231591 wait complete. 2023-07-14T11:11:42.006 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.c status 2023-07-14T11:11:42.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 -- 172.21.15.40:0/767455931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f87340ffae0 msgr2=0x7f87340ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 --2- 172.21.15.40:0/767455931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f87340ffae0 0x7f87340ffef0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f8724009990 tx=0x7f8724009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 -- 172.21.15.40:0/767455931 shutdown_connections 2023-07-14T11:11:42.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 --2- 172.21.15.40:0/767455931 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 0x7f8734107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 --2- 172.21.15.40:0/767455931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f87340ffae0 0x7f87340ffef0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 --2- 172.21.15.40:0/767455931 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8734100cd0 0x7f8734101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 -- 172.21.15.40:0/767455931 >> 172.21.15.40:0/767455931 conn(0x7f87340fb0e0 msgr2=0x7f87340fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 -- 172.21.15.40:0/767455931 shutdown_connections 2023-07-14T11:11:42.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.210+0000 7f873a7ff700 1 -- 172.21.15.40:0/767455931 wait complete. 2023-07-14T11:11:42.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 Processor -- start 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 -- start start 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f87340ffae0 0x7f87340742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 0x7f8734072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f87340735e0 con 0x7f8734100cd0 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8734073750 con 0x7f8734101660 2023-07-14T11:11:42.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.212+0000 7f873a7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f873419f920 con 0x7f87340ffae0 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f8733fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f87340ffae0 0x7f87340742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39854/0 (socket says 172.21.15.40:39854) 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 -- 172.21.15.40:0/982883877 learned_addr learned my addr 172.21.15.40:0/982883877 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f8738d9c700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 0x7f8734072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 -- 172.21.15.40:0/982883877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f87340ffae0 msgr2=0x7f87340742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f87340ffae0 0x7f87340742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 -- 172.21.15.40:0/982883877 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 msgr2=0x7f8734072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 0x7f8734072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 -- 172.21.15.40:0/982883877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f87240095f0 con 0x7f8734100cd0 2023-07-14T11:11:42.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f872bfff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f871c00d980 tx=0x7f871c00dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.213+0000 7f8731ffb700 1 -- 172.21.15.40:0/982883877 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f871c004220 con 0x7f8734100cd0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.214+0000 7f8731ffb700 1 -- 172.21.15.40:0/982883877 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f871c007640 con 0x7f8734100cd0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.214+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f873419fc00 con 0x7f8734100cd0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.214+0000 7f8731ffb700 1 -- 172.21.15.40:0/982883877 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f871c005690 con 0x7f8734100cd0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.214+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f87341a0160 con 0x7f8734100cd0 2023-07-14T11:11:42.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.217+0000 7f8731ffb700 1 -- 172.21.15.40:0/982883877 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f871c004750 con 0x7f8734100cd0 2023-07-14T11:11:42.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.218+0000 7f874e084b80 1 Processor -- start 2023-07-14T11:11:42.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.218+0000 7f874e084b80 1 -- start start 2023-07-14T11:11:42.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.218+0000 7f8731ffb700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 0x7f87200a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.218+0000 7f874e084b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f874e084b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac91e690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f874e084b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ec20 0x560aac91f090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f874e084b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac822c60 con 0x560aac91ddd0 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f874e084b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac8759f0 con 0x560aac91d480 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f874e084b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac86ee80 con 0x560aac91ec20 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8731ffb700 1 -- 172.21.15.40:0/982883877 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f871c0cca10 con 0x7f8734100cd0 2023-07-14T11:11:42.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8738d9c700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 0x7f87200a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8733fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac91e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8733fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54934/0 (socket says 172.21.15.40:54934) 2023-07-14T11:11:42.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8733fff700 1 -- 172.21.15.40:0/3595854271 learned_addr learned my addr 172.21.15.40:0/3595854271 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8738d9c700 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ec20 0x560aac91f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 -- 172.21.15.40:0/3595854271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ec20 msgr2=0x560aac91f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ec20 0x560aac91f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 -- 172.21.15.40:0/3595854271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 msgr2=0x560aac91d890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f872bfff700 1 -- 172.21.15.40:0/3595854271 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560aac69cd90 con 0x560aac91ddd0 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.219+0000 7f8733fff700 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f872bfff700 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac91e690 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f871c0ca160 tx=0x7f871c0994d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a3b8150a85fd29a9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f8738d9c700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 0x7f87200a3680 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f872c009ec0 tx=0x7f872c0134a0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f872a7fc700 1 -- 172.21.15.40:0/3595854271 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f871c09daa0 con 0x560aac91ddd0 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f872a7fc700 1 -- 172.21.15.40:0/3595854271 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f871c0a1ec0 con 0x560aac91ddd0 2023-07-14T11:11:42.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f872a7fc700 1 -- 172.21.15.40:0/3595854271 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f871c099cd0 con 0x560aac91ddd0 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f874e084b80 1 -- 172.21.15.40:0/3595854271 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 msgr2=0x560aac91e690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.220+0000 7f874e084b80 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac91e690 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f871c0ca160 tx=0x7f871c0994d0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 -- 172.21.15.40:0/3595854271 shutdown_connections 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ec20 0x560aac91f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac91d890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 --2- 172.21.15.40:0/3595854271 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac91e690 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 -- 172.21.15.40:0/3595854271 >> 172.21.15.40:0/3595854271 conn(0x560aac912d50 msgr2=0x560aac918180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 -- 172.21.15.40:0/3595854271 shutdown_connections 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 -- 172.21.15.40:0/3595854271 wait complete. 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 -- start start 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.221+0000 7f874e084b80 1 Processor -- start 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac926f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 0x560aac9274b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac80ea70 con 0x560aac91ec20 2023-07-14T11:11:42.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac80e770 con 0x560aac91ddd0 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac6f28d0 con 0x560aac91d480 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f874e084b80 10 client.0 _ll_register_callbacks cb 0x560aac80c710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f8738d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f8738d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39876/0 (socket says 172.21.15.40:39876) 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f8738d9c700 1 -- 172.21.15.40:0/4196692705 learned_addr learned my addr 172.21.15.40:0/4196692705 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f872bfff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 0x560aac9274b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.222+0000 7f8733fff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac926f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 -- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 msgr2=0x560aac926f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac926f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 -- 172.21.15.40:0/4196692705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 msgr2=0x560aac92e4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 -- 172.21.15.40:0/4196692705 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8734108850 con 0x560aac91ddd0 2023-07-14T11:11:42.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f8738d9c700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f872bfff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 0x560aac9274b0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f871c007e80 tx=0x7f871c000bc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d7f8926aa0ca820 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f87137fe700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:11:42.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f871c099170 con 0x560aac91ddd0 2023-07-14T11:11:42.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f871c099cd0 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560aac7c84a0 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f871c0b2280 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f874e084b80 10 client.25343 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.223+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f87341a0160 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560aac7c84a0 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f871c05ed50 con 0x560aac91ddd0 2023-07-14T11:11:42.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 10 client.25343 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 10 client.25343 resolve_mds: resolved name 'c' to daemon mds.c 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 20 client.25343 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac932300 0x560aac931420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 4 client.25343 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560aac867e60 con 0x560aac932300 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.224+0000 7f8733fff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac932300 0x560aac931420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.225+0000 7f8733fff700 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac932300 0x560aac931420 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=c87e53ef997b2fe0 server_cookie=2d41bf34b9d9d465 in_seq=0 out_seq=0 2023-07-14T11:11:42.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.225+0000 7f87137fe700 10 client.25343 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f87137fe700 1 -- 172.21.15.40:0/4196692705 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x560aac867e60 con 0x560aac932300 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f87137fe700 10 client.25343 handle_command_reply: tid=0 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 client.25343 shutdown 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 msgr2=0x560aac9274b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 0x560aac9274b0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f871c007e80 tx=0x7f871c000bc0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 shutdown_connections 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac932300 0x560aac931420 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac926f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f87137fe700 0 client.25343 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 --2- 172.21.15.40:0/4196692705 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ddd0 0x560aac9274b0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 >> 172.21.15.40:0/4196692705 conn(0x560aac912d50 msgr2=0x560aac917860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 shutdown_connections 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 1 -- 172.21.15.40:0/4196692705 wait complete. 2023-07-14T11:11:42.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.226+0000 7f874e084b80 20 client.25343 trim_cache size 0 max 16384 2023-07-14T11:11:42.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 Processor -- start 2023-07-14T11:11:42.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 -- start start 2023-07-14T11:11:42.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ddd0 0x560aac921fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac918e80 con 0x560aac91ec20 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac919660 con 0x560aac91d480 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.245+0000 7f874e084b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac822c60 con 0x560aac91ddd0 2023-07-14T11:11:42.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8738d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:54940/0 (socket says 172.21.15.40:54940) 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 -- 172.21.15.40:0/436823922 learned_addr learned my addr 172.21.15.40:0/436823922 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f872bfff700 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ddd0 0x560aac921fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 -- 172.21.15.40:0/436823922 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ddd0 msgr2=0x560aac921fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ddd0 0x560aac921fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 -- 172.21.15.40:0/436823922 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 msgr2=0x560aac92e4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 -- 172.21.15.40:0/436823922 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560aac925fb0 con 0x560aac91d480 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.246+0000 7f8733fff700 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f8724000c00 tx=0x7f8724005960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=17d12afd6b767fdc server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f872affd700 1 -- 172.21.15.40:0/436823922 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872402a070 con 0x560aac91d480 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f872affd700 1 -- 172.21.15.40:0/436823922 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f872401e040 con 0x560aac91d480 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f872affd700 1 -- 172.21.15.40:0/436823922 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872401d040 con 0x560aac91d480 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 -- 172.21.15.40:0/436823922 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 msgr2=0x560aac921a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f8724000c00 tx=0x7f8724005960 comp rx=0 tx=0).stop 2023-07-14T11:11:42.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 -- 172.21.15.40:0/436823922 shutdown_connections 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ec20 0x560aac92e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91d480 0x560aac921a80 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 --2- 172.21.15.40:0/436823922 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91ddd0 0x560aac921fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 -- 172.21.15.40:0/436823922 >> 172.21.15.40:0/436823922 conn(0x560aac912d50 msgr2=0x560aac931420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.247+0000 7f874e084b80 1 -- 172.21.15.40:0/436823922 shutdown_connections 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.248+0000 7f874e084b80 1 -- 172.21.15.40:0/436823922 wait complete. 2023-07-14T11:11:42.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.248+0000 7f874e084b80 1 -- start start 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.248+0000 7f874e084b80 1 Processor -- start 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac9291f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ec20 0x560aac985b90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac92e760 con 0x560aac91ddd0 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac92e8d0 con 0x560aac91ec20 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560aac9860d0 con 0x560aac91d480 2023-07-14T11:11:42.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f874e084b80 10 client.0 _ll_register_callbacks cb 0x7f872c0097c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f872bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f8733fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac9291f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f872bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39900/0 (socket says 172.21.15.40:39900) 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f872bfff700 1 -- 172.21.15.40:0/1175277970 learned_addr learned my addr 172.21.15.40:0/1175277970 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.249+0000 7f8738d9c700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ec20 0x560aac985b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 -- 172.21.15.40:0/1175277970 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ec20 msgr2=0x560aac985b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ec20 0x560aac985b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 -- 172.21.15.40:0/1175277970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 msgr2=0x560aac929730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 -- 172.21.15.40:0/1175277970 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8724005480 con 0x560aac91d480 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f872bfff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f8733fff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac9291f0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f87240059d0 tx=0x7f87240057c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8930ea56b01058c5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f87137fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f872402a040 con 0x560aac91d480 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f871c0a1ec0 con 0x560aac91d480 2023-07-14T11:11:42.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.250+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f872400bd80 con 0x560aac91d480 2023-07-14T11:11:42.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f872401d040 con 0x560aac91d480 2023-07-14T11:11:42.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 10 client.25354 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x560aac91f2c0 con 0x560aac91d480 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f871c0a1ec0 con 0x560aac91d480 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f872400f720 con 0x560aac91d480 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 10 client.25354 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 10 client.25354 resolve_mds: resolved name 'c' to daemon mds.c 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 20 client.25354 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac98a950 0x560aac9872f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 4 client.25354 mds_command: new command op to 24523 tid=0[{"prefix": "status"}] 2023-07-14T11:11:42.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "status"}) v1 -- 0x560aac82a000 con 0x560aac98a950 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.251+0000 7f872bfff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac98a950 0x560aac9872f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f872bfff700 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac98a950 0x560aac9872f0 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e44cfb7e62f5e8bc server_cookie=a26c5947e50418f6 in_seq=0 out_seq=0 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f87137fe700 10 client.25354 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f87137fe700 1 -- 172.21.15.40:0/1175277970 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+327 (crc 0 0 0) 0x560aac82a000 con 0x560aac98a950 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f87137fe700 10 client.25354 handle_command_reply: tid=0 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 client.25354 shutdown 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 msgr2=0x560aac9291f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac9291f0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f87240059d0 tx=0x7f87240057c0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 shutdown_connections 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x560aac98a950 0x560aac9872f0 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560aac91ec20 0x560aac985b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560aac91d480 0x560aac9291f0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 --2- 172.21.15.40:0/1175277970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560aac91ddd0 0x560aac929730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 >> 172.21.15.40:0/1175277970 conn(0x560aac912d50 msgr2=0x560aac921a80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f87137fe700 0 client.25354 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.252+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 shutdown_connections 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f874e084b80 1 -- 172.21.15.40:0/1175277970 wait complete. 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f874e084b80 20 client.25354 trim_cache size 0 max 16384 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:42.262 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 2, 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24523, 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 87.904110106999994, 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:42.263 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:42.264 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 137.182043263 2023-07-14T11:11:42.264 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:42.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 msgr2=0x7f87200a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 0x7f87200a3680 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f872c009ec0 tx=0x7f872c0134a0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 msgr2=0x7f8734072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f871c00d980 tx=0x7f871c00dc90 comp rx=0 tx=0).stop 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 shutdown_connections 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f87200a11d0 0x7f87200a3680 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8734101660 0x7f8734072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f87340ffae0 0x7f87340742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 --2- 172.21.15.40:0/982883877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8734100cd0 0x7f8734072900 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 >> 172.21.15.40:0/982883877 conn(0x7f87340fb0e0 msgr2=0x7f87341041b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 shutdown_connections 2023-07-14T11:11:42.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.253+0000 7f873a7ff700 1 -- 172.21.15.40:0/982883877 wait complete. 2023-07-14T11:11:42.273 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.a status 2023-07-14T11:11:42.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 -- 172.21.15.40:0/3689349743 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c0ffae0 msgr2=0x7faa2c0ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 --2- 172.21.15.40:0/3689349743 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c0ffae0 0x7faa2c0ffef0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7faa20009990 tx=0x7faa20009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 -- 172.21.15.40:0/3689349743 shutdown_connections 2023-07-14T11:11:42.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 --2- 172.21.15.40:0/3689349743 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 --2- 172.21.15.40:0/3689349743 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c0ffae0 0x7faa2c0ffef0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 --2- 172.21.15.40:0/3689349743 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c100cd0 0x7faa2c101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 -- 172.21.15.40:0/3689349743 >> 172.21.15.40:0/3689349743 conn(0x7faa2c0fb0e0 msgr2=0x7faa2c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 -- 172.21.15.40:0/3689349743 shutdown_connections 2023-07-14T11:11:42.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.477+0000 7faa32fff700 1 -- 172.21.15.40:0/3689349743 wait complete. 2023-07-14T11:11:42.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 Processor -- start 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 -- start start 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c0ffae0 0x7faa2c197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c100cd0 0x7faa2c198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa2c19f660 con 0x7faa2c101660 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa2c19f7a0 con 0x7faa2c100cd0 2023-07-14T11:11:42.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.478+0000 7faa32fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa2c19f930 con 0x7faa2c0ffae0 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39926/0 (socket says 172.21.15.40:39926) 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 -- 172.21.15.40:0/4258163868 learned_addr learned my addr 172.21.15.40:0/4258163868 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa2bfff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c100cd0 0x7faa2c198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa30d9b700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c0ffae0 0x7faa2c197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 -- 172.21.15.40:0/4258163868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c0ffae0 msgr2=0x7faa2c197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c0ffae0 0x7faa2c197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 -- 172.21.15.40:0/4258163868 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c100cd0 msgr2=0x7faa2c198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c100cd0 0x7faa2c198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 -- 172.21.15.40:0/4258163868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa200095f0 con 0x7faa2c101660 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.479+0000 7faa3159c700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7faa2400b970 tx=0x7faa2400bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.480+0000 7faa29ffb700 1 -- 172.21.15.40:0/4258163868 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa2400d860 con 0x7faa2c101660 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.480+0000 7faa29ffb700 1 -- 172.21.15.40:0/4258163868 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faa2400de80 con 0x7faa2c101660 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.480+0000 7faa29ffb700 1 -- 172.21.15.40:0/4258163868 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa24017400 con 0x7faa2c101660 2023-07-14T11:11:42.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.480+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faa2c19fc10 con 0x7faa2c101660 2023-07-14T11:11:42.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.480+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa2c1a0120 con 0x7faa2c101660 2023-07-14T11:11:42.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.483+0000 7faa29ffb700 1 -- 172.21.15.40:0/4258163868 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7faa2400e630 con 0x7faa2c101660 2023-07-14T11:11:42.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.484+0000 7faa29ffb700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 0x7faa1c0a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.484+0000 7faa46840b80 1 Processor -- start 2023-07-14T11:11:42.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.484+0000 7faa29ffb700 1 -- 172.21.15.40:0/4258163868 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faa24006e40 con 0x7faa2c101660 2023-07-14T11:11:42.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 -- start start 2023-07-14T11:11:42.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa30d9b700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 0x7faa1c0a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d057690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 0x557b3d058090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cf5bc60 con 0x557b3d056480 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cfae9f0 con 0x557b3d057c20 2023-07-14T11:11:42.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa46840b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cfa7e80 con 0x557b3d056dd0 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa30d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa3159c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 0x557b3d058090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa30d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39934/0 (socket says 172.21.15.40:39934) 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa30d9b700 1 -- 172.21.15.40:0/702756096 learned_addr learned my addr 172.21.15.40:0/702756096 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.485+0000 7faa2bfff700 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d057690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 0x7faa1c0a3680 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7faa2000b2e0 tx=0x7faa20005290 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 -- 172.21.15.40:0/702756096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 msgr2=0x557b3d057690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d057690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 -- 172.21.15.40:0/702756096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 msgr2=0x557b3d058090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 0x557b3d058090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 -- 172.21.15.40:0/702756096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557b3cdd5d90 con 0x557b3d056480 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa3159c700 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 0x557b3d058090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa30d9b700 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7faa20015000 tx=0x7faa20028000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b4ffa71c246c68f1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa16ffd700 1 -- 172.21.15.40:0/702756096 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa200045c0 con 0x557b3d056480 2023-07-14T11:11:42.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa16ffd700 1 -- 172.21.15.40:0/702756096 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faa2001e040 con 0x557b3d056480 2023-07-14T11:11:42.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.486+0000 7faa16ffd700 1 -- 172.21.15.40:0/702756096 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa2002fb70 con 0x557b3d056480 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 -- 172.21.15.40:0/702756096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 msgr2=0x557b3d056890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7faa20015000 tx=0x7faa20028000 comp rx=0 tx=0).stop 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 -- 172.21.15.40:0/702756096 shutdown_connections 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d057c20 0x557b3d058090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d056480 0x557b3d056890 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 --2- 172.21.15.40:0/702756096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d057690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 -- 172.21.15.40:0/702756096 >> 172.21.15.40:0/702756096 conn(0x557b3d04bd50 msgr2=0x557b3d051180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 -- 172.21.15.40:0/702756096 shutdown_connections 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.487+0000 7faa46840b80 1 -- 172.21.15.40:0/702756096 wait complete. 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.488+0000 7faa46840b80 1 -- start start 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.488+0000 7faa46840b80 1 Processor -- start 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d060000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d060540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cf47a70 con 0x557b3d057c20 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cf47770 con 0x557b3d056480 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3ce2b8d0 con 0x557b3d056dd0 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa46840b80 10 client.0 _ll_register_callbacks cb 0x557b3cf45710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39948/0 (socket says 172.21.15.40:39948) 2023-07-14T11:11:42.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa3159c700 1 -- 172.21.15.40:0/114603572 learned_addr learned my addr 172.21.15.40:0/114603572 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa2bfff700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d060540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.489+0000 7faa30d9b700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d060000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 -- 172.21.15.40:0/114603572 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 msgr2=0x557b3d060540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d060540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 -- 172.21.15.40:0/114603572 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 msgr2=0x557b3d060000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d060000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 -- 172.21.15.40:0/114603572 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa2c1a0120 con 0x557b3d057c20 2023-07-14T11:11:42.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa3159c700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7faa240997b0 tx=0x7faa24004be0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c52c4e18aaeb993 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa0bfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa24099c80 con 0x557b3d057c20 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faa24099820 con 0x557b3d057c20 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557b3cf014a0 con 0x557b3d057c20 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7faa2409d850 con 0x557b3d057c20 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa46840b80 10 client.15363 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.490+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7faa2c108850 con 0x557b3d057c20 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557b3cf014a0 con 0x557b3d057c20 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7faa240a17e0 con 0x557b3d057c20 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 10 client.15363 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 10 client.15363 resolve_mds: resolved name 'a' to daemon mds.a 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 20 client.15363 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d06b2a0 0x557b3d06a690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 4 client.15363 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:42.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557b3cfa0e60 con 0x557b3d06b2a0 2023-07-14T11:11:42.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.491+0000 7faa30d9b700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d06b2a0 0x557b3d06a690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.492+0000 7faa30d9b700 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d06b2a0 0x557b3d06a690 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2e6d6ee7eb6379fc server_cookie=5ec93e9ce137d835 in_seq=0 out_seq=0 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.492+0000 7faa0bfff700 10 client.15363 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.492+0000 7faa0bfff700 1 -- 172.21.15.40:0/114603572 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x557b3cfa0e60 con 0x557b3d06b2a0 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.492+0000 7faa0bfff700 10 client.15363 handle_command_reply: tid=0 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.492+0000 7faa46840b80 1 client.15363 shutdown 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 msgr2=0x557b3d067580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7faa240997b0 tx=0x7faa24004be0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 shutdown_connections 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d06b2a0 0x557b3d06a690 crc :-1 s=READY pgs=21 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d067580 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d060000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 --2- 172.21.15.40:0/114603572 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d060540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa0bfff700 0 client.15363 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 >> 172.21.15.40:0/114603572 conn(0x557b3d04bd50 msgr2=0x557b3d050860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 shutdown_connections 2023-07-14T11:11:42.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 1 -- 172.21.15.40:0/114603572 wait complete. 2023-07-14T11:11:42.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.493+0000 7faa46840b80 20 client.15363 trim_cache size 0 max 16384 2023-07-14T11:11:42.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.511+0000 7faa46840b80 1 Processor -- start 2023-07-14T11:11:42.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 -- start start 2023-07-14T11:11:42.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05eca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3d0611f0 con 0x557b3d057c20 2023-07-14T11:11:42.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3cf76790 con 0x557b3d056480 2023-07-14T11:11:42.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa46840b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3ce37de0 con 0x557b3d056dd0 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa30d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05eca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39964/0 (socket says 172.21.15.40:39964) 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 -- 172.21.15.40:0/3080674923 learned_addr learned my addr 172.21.15.40:0/3080674923 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa2bfff700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 -- 172.21.15.40:0/3080674923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 msgr2=0x557b3d067540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 -- 172.21.15.40:0/3080674923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 msgr2=0x557b3d05eca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05eca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa3159c700 1 -- 172.21.15.40:0/3080674923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557b3cdbec10 con 0x557b3d057c20 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa2bfff700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.512+0000 7faa30d9b700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05eca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.513+0000 7faa3159c700 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7faa24095d20 tx=0x7faa24097dd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a7ed4f077a0c681c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.513+0000 7faa177fe700 1 -- 172.21.15.40:0/3080674923 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa24099820 con 0x557b3d057c20 2023-07-14T11:11:42.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.513+0000 7faa177fe700 1 -- 172.21.15.40:0/3080674923 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faa2409d850 con 0x557b3d057c20 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.513+0000 7faa177fe700 1 -- 172.21.15.40:0/3080674923 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa2400d040 con 0x557b3d057c20 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- 172.21.15.40:0/3080674923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 msgr2=0x557b3d06a690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7faa24095d20 tx=0x7faa24097dd0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- 172.21.15.40:0/3080674923 shutdown_connections 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d06a690 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05eca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 --2- 172.21.15.40:0/3080674923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- 172.21.15.40:0/3080674923 >> 172.21.15.40:0/3080674923 conn(0x557b3d04bd50 msgr2=0x557b3d05e010 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- 172.21.15.40:0/3080674923 shutdown_connections 2023-07-14T11:11:42.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- 172.21.15.40:0/3080674923 wait complete. 2023-07-14T11:11:42.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 -- start start 2023-07-14T11:11:42.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.514+0000 7faa46840b80 1 Processor -- start 2023-07-14T11:11:42.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05c190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3d05cf70 con 0x557b3d057c20 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3d0bf120 con 0x557b3d056480 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557b3d0bf260 con 0x557b3d056dd0 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa46840b80 10 client.0 _ll_register_callbacks cb 0x7faa240995f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39972/0 (socket says 172.21.15.40:39972) 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 -- 172.21.15.40:0/3086995602 learned_addr learned my addr 172.21.15.40:0/3086995602 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa30d9b700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05c190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa2bfff700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 -- 172.21.15.40:0/3086995602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 msgr2=0x557b3d067540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 -- 172.21.15.40:0/3086995602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 msgr2=0x557b3d05c190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05c190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.515+0000 7faa3159c700 1 -- 172.21.15.40:0/3086995602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557b3cdd5d90 con 0x557b3d057c20 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa3159c700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7faa2400efd0 tx=0x7faa2400e000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9070c1a073af7999 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa0bfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faa24099c80 con 0x557b3d057c20 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faa240c7030 con 0x557b3d057c20 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7faa2001e040 con 0x557b3d057c20 2023-07-14T11:11:42.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7faa24099820 con 0x557b3d057c20 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa46840b80 10 client.15369 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7faa2c108850 con 0x557b3d057c20 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.516+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faa2001e040 con 0x557b3d057c20 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7faa24098880 con 0x557b3d057c20 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 10 client.15369 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 10 client.15369 resolve_mds: resolved name 'a' to daemon mds.a 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 20 client.15369 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d0c3a70 0x557b3d0c0460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 4 client.15369 mds_command: new command op to 24565 tid=0[{"prefix": "status"}] 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "status"}) v1 -- 0x557b3cf63000 con 0x557b3d0c3a70 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa30d9b700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d0c3a70 0x557b3d0c0460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa0bfff700 10 client.15369 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.517+0000 7faa30d9b700 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d0c3a70 0x557b3d0c0460 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=2ab58963bdc8505e server_cookie=f9f0db6dffec8a4f in_seq=0 out_seq=0 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa0bfff700 1 -- 172.21.15.40:0/3086995602 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+332 (crc 0 0 0) 0x557b3cf63000 con 0x557b3d0c3a70 2023-07-14T11:11:42.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa0bfff700 10 client.15369 handle_command_reply: tid=0 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 client.15369 shutdown 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 msgr2=0x557b3d05c830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7faa2400efd0 tx=0x7faa2400e000 comp rx=0 tx=0).stop 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 shutdown_connections 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x557b3d0c3a70 0x557b3d0c0460 crc :-1 s=READY pgs=22 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557b3d057c20 0x557b3d05c830 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557b3d056480 0x557b3d05c190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa0bfff700 0 client.15369 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 --2- 172.21.15.40:0/3086995602 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557b3d056dd0 0x557b3d067540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 >> 172.21.15.40:0/3086995602 conn(0x557b3d04bd50 msgr2=0x557b3d05eca0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 shutdown_connections 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 1 -- 172.21.15.40:0/3086995602 wait complete. 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.518+0000 7faa46840b80 20 client.15369 trim_cache size 0 max 16384 2023-07-14T11:11:42.528 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 1, 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24565, 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:42.529 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 88.745241128000004, 2023-07-14T11:11:42.530 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:42.530 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:42.530 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:42.530 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 126.67988837199999 2023-07-14T11:11:42.530 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 msgr2=0x7faa1c0a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 0x7faa1c0a3680 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7faa2000b2e0 tx=0x7faa20005290 comp rx=0 tx=0).stop 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 msgr2=0x7faa2c19f090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7faa2400b970 tx=0x7faa2400bc80 comp rx=0 tx=0).stop 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 shutdown_connections 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faa1c0a11d0 0x7faa1c0a3680 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faa2c101660 0x7faa2c19f090 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faa2c0ffae0 0x7faa2c197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 --2- 172.21.15.40:0/4258163868 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faa2c100cd0 0x7faa2c198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 >> 172.21.15.40:0/4258163868 conn(0x7faa2c0fb0e0 msgr2=0x7faa2c0fd3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 shutdown_connections 2023-07-14T11:11:42.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.519+0000 7faa32fff700 1 -- 172.21.15.40:0/4258163868 wait complete. 2023-07-14T11:11:42.536 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:42 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3014911733' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:42.538 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.b status 2023-07-14T11:11:42.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 -- 172.21.15.40:0/4109171402 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac8101660 msgr2=0x7faac8107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 --2- 172.21.15.40:0/4109171402 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac8101660 0x7faac8107f20 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7faac000b0a0 tx=0x7faac000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 -- 172.21.15.40:0/4109171402 shutdown_connections 2023-07-14T11:11:42.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 --2- 172.21.15.40:0/4109171402 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac8101660 0x7faac8107f20 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 --2- 172.21.15.40:0/4109171402 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac80ffae0 0x7faac80ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 --2- 172.21.15.40:0/4109171402 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 0x7faac8101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 -- 172.21.15.40:0/4109171402 >> 172.21.15.40:0/4109171402 conn(0x7faac80fb0e0 msgr2=0x7faac80fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 -- 172.21.15.40:0/4109171402 shutdown_connections 2023-07-14T11:11:42.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.745+0000 7faacc9ff700 1 -- 172.21.15.40:0/4109171402 wait complete. 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.746+0000 7faacc9ff700 1 Processor -- start 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.746+0000 7faacc9ff700 1 -- start start 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.746+0000 7faacc9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.746+0000 7faacc9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 0x7faac8198090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faacc9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 0x7faac819f100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faacc9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faac819f6d0 con 0x7faac80ffae0 2023-07-14T11:11:42.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faacc9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faac819f810 con 0x7faac8101660 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faacc9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faac819f9a0 con 0x7faac8100cd0 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40000/0 (socket says 172.21.15.40:40000) 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 -- 172.21.15.40:0/359187228 learned_addr learned my addr 172.21.15.40:0/359187228 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac6d9d700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 0x7faac819f100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faabffff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 0x7faac8198090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 -- 172.21.15.40:0/359187228 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 msgr2=0x7faac8198090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 0x7faac8198090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 -- 172.21.15.40:0/359187228 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 msgr2=0x7faac819f100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 0x7faac819f100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac659c700 1 -- 172.21.15.40:0/359187228 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faab8009520 con 0x7faac80ffae0 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.747+0000 7faac6d9d700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 0x7faac819f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faac659c700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7faab800ea50 tx=0x7faab800ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faabf7fe700 1 -- 172.21.15.40:0/359187228 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab800cbd0 con 0x7faac80ffae0 2023-07-14T11:11:42.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faabf7fe700 1 -- 172.21.15.40:0/359187228 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faab800cd30 con 0x7faac80ffae0 2023-07-14T11:11:42.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faabf7fe700 1 -- 172.21.15.40:0/359187228 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab8018680 con 0x7faac80ffae0 2023-07-14T11:11:42.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faac0009d00 con 0x7faac80ffae0 2023-07-14T11:11:42.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.748+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faac819ffb0 con 0x7faac80ffae0 2023-07-14T11:11:42.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.752+0000 7faabf7fe700 1 -- 172.21.15.40:0/359187228 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7faab8010430 con 0x7faac80ffae0 2023-07-14T11:11:42.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.753+0000 7faae0274b80 1 Processor -- start 2023-07-14T11:11:42.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.753+0000 7faae0274b80 1 -- start start 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.753+0000 7faae0274b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46d3890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.753+0000 7faae0274b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46d4690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faae0274b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46d5090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faae0274b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d45d8c60 con 0x5621d46d4c20 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faae0274b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d462b9f0 con 0x5621d46d3dd0 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faae0274b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d4624e80 con 0x5621d46d3480 2023-07-14T11:11:42.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faabf7fe700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 0x7faab40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faabffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46d5090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faabffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46d5090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40016/0 (socket says 172.21.15.40:40016) 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faabf7fe700 1 -- 172.21.15.40:0/359187228 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faab8014070 con 0x7faac80ffae0 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faabffff700 1 -- 172.21.15.40:0/2102359367 learned_addr learned my addr 172.21.15.40:0/2102359367 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46d4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac6d9d700 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46d3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac6d9d700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 0x7faab40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 -- 172.21.15.40:0/2102359367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 msgr2=0x5621d46d3890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.759 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46d3890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 -- 172.21.15.40:0/2102359367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 msgr2=0x5621d46d5090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46d5090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.754+0000 7faac659c700 1 -- 172.21.15.40:0/2102359367 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5621d4452d90 con 0x5621d46d3dd0 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.755+0000 7faac659c700 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46d4690 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7faab8099e30 tx=0x7faab8010c80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2590c0496e3ddb94 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.755+0000 7faac6d9d700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 0x7faab40a3680 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7faac000b780 tx=0x7faac0006210 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.755+0000 7faabcff9700 1 -- 172.21.15.40:0/2102359367 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab80a2580 con 0x5621d46d3dd0 2023-07-14T11:11:42.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.755+0000 7faabcff9700 1 -- 172.21.15.40:0/2102359367 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faab80ca930 con 0x5621d46d3dd0 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.755+0000 7faabcff9700 1 -- 172.21.15.40:0/2102359367 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab80106e0 con 0x5621d46d3dd0 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- 172.21.15.40:0/2102359367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 msgr2=0x5621d46d4690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46d4690 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7faab8099e30 tx=0x7faab8010c80 comp rx=0 tx=0).stop 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- 172.21.15.40:0/2102359367 shutdown_connections 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46d5090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46d3890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 --2- 172.21.15.40:0/2102359367 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46d4690 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- 172.21.15.40:0/2102359367 >> 172.21.15.40:0/2102359367 conn(0x5621d46c8d50 msgr2=0x5621d46ce180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- 172.21.15.40:0/2102359367 shutdown_connections 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- 172.21.15.40:0/2102359367 wait complete. 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 -- start start 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.756+0000 7faae0274b80 1 Processor -- start 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 0x5621d46dd5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d4c20 0x5621d46e45f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d45c4a70 con 0x5621d46d3480 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d45c4770 con 0x5621d46d4c20 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d44a88d0 con 0x5621d46d3dd0 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faae0274b80 10 client.0 _ll_register_callbacks cb 0x5621d45c2710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faac659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faabffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 0x5621d46dd5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faac659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40018/0 (socket says 172.21.15.40:40018) 2023-07-14T11:11:42.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faac659c700 1 -- 172.21.15.40:0/590163170 learned_addr learned my addr 172.21.15.40:0/590163170 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.757+0000 7faac6d9d700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d4c20 0x5621d46e45f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 -- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d4c20 msgr2=0x5621d46e45f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d4c20 0x5621d46e45f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 -- 172.21.15.40:0/590163170 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 msgr2=0x5621d46dd070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 -- 172.21.15.40:0/590163170 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faac819ffb0 con 0x5621d46d3dd0 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faac659c700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faabffff700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 0x5621d46dd5b0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7faab000bb40 tx=0x7faab000be50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c72287f0bb8eb9c5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faaadffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:11:42.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab0016070 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5621d457e4a0 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.758+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faab0004530 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7faab0003d00 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 10 client.25384 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5621d443bc10 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5621d457e4a0 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7faab0011560 con 0x5621d46d3dd0 2023-07-14T11:11:42.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 10 client.25384 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 10 client.25384 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 20 client.25384 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d46e8380 0x5621d46e7750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 4 client.25384 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.759+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5621d461de60 con 0x5621d46e8380 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.760+0000 7faac659c700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d46e8380 0x5621d46e7750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.760+0000 7faac659c700 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d46e8380 0x5621d46e7750 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=43e1b5e51f540f6c server_cookie=4ebf68a2b5e0bdd8 in_seq=0 out_seq=0 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.760+0000 7faaadffb700 10 client.25384 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.761+0000 7faaadffb700 1 -- 172.21.15.40:0/590163170 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5621d461de60 con 0x5621d46e8380 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.761+0000 7faaadffb700 10 client.25384 handle_command_reply: tid=0 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.761+0000 7faae0274b80 1 client.25384 shutdown 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.761+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 msgr2=0x5621d46dd5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.761+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 0x5621d46dd5b0 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7faab000bb40 tx=0x7faab000be50 comp rx=0 tx=0).stop 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 shutdown_connections 2023-07-14T11:11:42.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d46e8380 0x5621d46e7750 crc :-1 s=READY pgs=28 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d4c20 0x5621d46e45f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d3480 0x5621d46dd070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faaadffb700 0 client.25384 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 --2- 172.21.15.40:0/590163170 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3dd0 0x5621d46dd5b0 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 >> 172.21.15.40:0/590163170 conn(0x5621d46c8d50 msgr2=0x5621d46cd860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 shutdown_connections 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 1 -- 172.21.15.40:0/590163170 wait complete. 2023-07-14T11:11:42.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.762+0000 7faae0274b80 20 client.25384 trim_cache size 0 max 16384 2023-07-14T11:11:42.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 Processor -- start 2023-07-14T11:11:42.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 -- start start 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46e45b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d46d4520 con 0x5621d46d4c20 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d46de260 con 0x5621d46d3dd0 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.780+0000 7faae0274b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d45f3790 con 0x5621d46d3480 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47632/0 (socket says 172.21.15.40:47632) 2023-07-14T11:11:42.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 -- 172.21.15.40:0/3611767353 learned_addr learned my addr 172.21.15.40:0/3611767353 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac6d9d700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40022/0 (socket says 172.21.15.40:40022) 2023-07-14T11:11:42.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faabffff700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46e45b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 -- 172.21.15.40:0/3611767353 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 msgr2=0x5621d46e45b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46e45b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 -- 172.21.15.40:0/3611767353 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 msgr2=0x5621d46e7750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 -- 172.21.15.40:0/3611767353 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5621d46d52c0 con 0x5621d46d3480 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac6d9d700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.781+0000 7faac659c700 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7faab8000c00 tx=0x7faab800ef80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9383d4d91a08f495 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faabd7fa700 1 -- 172.21.15.40:0/3611767353 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab80b8070 con 0x5621d46d3480 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faabd7fa700 1 -- 172.21.15.40:0/3611767353 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faab80a2600 con 0x5621d46d3480 2023-07-14T11:11:42.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faabd7fa700 1 -- 172.21.15.40:0/3611767353 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faab801eb90 con 0x5621d46d3480 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 -- 172.21.15.40:0/3611767353 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 msgr2=0x5621d46dbd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7faab8000c00 tx=0x7faab800ef80 comp rx=0 tx=0).stop 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 -- 172.21.15.40:0/3611767353 shutdown_connections 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d46e7750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46dbd70 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 --2- 172.21.15.40:0/3611767353 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46e45b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 -- 172.21.15.40:0/3611767353 >> 172.21.15.40:0/3611767353 conn(0x5621d46c8d50 msgr2=0x5621d46cd860 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 -- 172.21.15.40:0/3611767353 shutdown_connections 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.782+0000 7faae0274b80 1 -- 172.21.15.40:0/3611767353 wait complete. 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 -- start start 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 Processor -- start 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46df820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46dfd60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d46e4780 con 0x5621d46d4c20 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d46e48f0 con 0x5621d46d3dd0 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5621d473c760 con 0x5621d46d3480 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.783+0000 7faae0274b80 10 client.0 _ll_register_callbacks cb 0x5621d45c2710 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:40024/0 (socket says 172.21.15.40:40024) 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 -- 172.21.15.40:0/408561648 learned_addr learned my addr 172.21.15.40:0/408561648 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faabffff700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46dfd60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac659c700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46df820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 -- 172.21.15.40:0/408561648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 msgr2=0x5621d46df820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46df820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 -- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 msgr2=0x5621d46dfd60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46dfd60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 -- 172.21.15.40:0/408561648 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faac8108850 con 0x5621d46d4c20 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faabffff700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46dfd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faac6d9d700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7faac0015000 tx=0x7faac000b3b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c338d866052e1839 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:42.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.784+0000 7faaadffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:42.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faac000e070 con 0x5621d46d4c20 2023-07-14T11:11:42.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7faab80ca930 con 0x5621d46d4c20 2023-07-14T11:11:42.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7faac0025040 con 0x5621d46d4c20 2023-07-14T11:11:42.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:42 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3014911733' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7faac0020ae0 con 0x5621d46d4c20 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faae0274b80 10 client.15387 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5621d4452d90 con 0x5621d46d4c20 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faab80ca930 con 0x5621d46d4c20 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.785+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7faac0005040 con 0x5621d46d4c20 2023-07-14T11:11:42.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 10 client.15387 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 10 client.15387 resolve_mds: resolved name 'b' to daemon mds.b 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 20 client.15387 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d4741570 0x5621d473dec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 4 client.15387 mds_command: new command op to 24569 tid=0[{"prefix": "status"}] 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "status"}) v1 -- 0x5621d45e0000 con 0x5621d4741570 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faac659c700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d4741570 0x5621d473dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faac659c700 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d4741570 0x5621d473dec0 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3db9ce23ec32c505 server_cookie=13a647e9c4dddd73 in_seq=0 out_seq=0 2023-07-14T11:11:42.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.786+0000 7faaadffb700 10 client.15387 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faaadffb700 1 -- 172.21.15.40:0/408561648 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+327 (crc 0 0 0) 0x5621d45e0000 con 0x5621d4741570 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faaadffb700 10 client.15387 handle_command_reply: tid=0 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 client.15387 shutdown 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 msgr2=0x5621d473c220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7faac0015000 tx=0x7faac000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 shutdown_connections 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5621d4741570 0x5621d473dec0 crc :-1 s=READY pgs=29 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5621d46d4c20 0x5621d473c220 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5621d46d3480 0x5621d46df820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faaadffb700 0 client.15387 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 --2- 172.21.15.40:0/408561648 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5621d46d3dd0 0x5621d46dfd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 >> 172.21.15.40:0/408561648 conn(0x5621d46c8d50 msgr2=0x5621d46dbd70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 shutdown_connections 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 1 -- 172.21.15.40:0/408561648 wait complete. 2023-07-14T11:11:42.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.787+0000 7faae0274b80 20 client.15387 trim_cache size 0 max 16384 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout: "cluster_fsid": "bbe0b002-2235-11ee-9b32-001a4aab830c", 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout: "whoami": 0, 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout: "id": 24569, 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout: "want_state": "up:active", 2023-07-14T11:11:42.798 INFO:teuthology.orchestra.run.smithi040.stdout: "state": "up:active", 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "fs_name": "cephfs", 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "rank_uptime": 119.17077272500001, 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "mdsmap_epoch": 27, 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch": 92, 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_epoch_barrier": 90, 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout: "uptime": 124.707478862 2023-07-14T11:11:42.799 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:42.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 msgr2=0x7faab40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 0x7faab40a3680 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7faac000b780 tx=0x7faac0006210 comp rx=0 tx=0).stop 2023-07-14T11:11:42.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 msgr2=0x7faac8197b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:42.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7faab800ea50 tx=0x7faab800ed60 comp rx=0 tx=0).stop 2023-07-14T11:11:42.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 shutdown_connections 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7faab40a11d0 0x7faab40a3680 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faac8101660 0x7faac819f100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faac80ffae0 0x7faac8197b50 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 --2- 172.21.15.40:0/359187228 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faac8100cd0 0x7faac8198090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 >> 172.21.15.40:0/359187228 conn(0x7faac80fb0e0 msgr2=0x7faac80fd280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 shutdown_connections 2023-07-14T11:11:42.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:42.788+0000 7faacc9ff700 1 -- 172.21.15.40:0/359187228 wait complete. 2023-07-14T11:11:42.807 INFO:teuthology.run_tasks:Running task kclient... 2023-07-14T11:11:42.821 INFO:tasks.kclient:Mounting kernel clients... 2023-07-14T11:11:42.821 INFO:tasks.kclient:config is {'client.0': {}} 2023-07-14T11:11:42.821 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:42.822 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list 2023-07-14T11:11:42.822 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:42 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3014911733' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:42.865 INFO:teuthology.orchestra.run.smithi040.stdout:ceph-ns--home-ubuntu-cephtest-mnt.admin (id: 0) 2023-07-14T11:11:42.866 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:42.866 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns delete ceph-ns--home-ubuntu-cephtest-mnt.admin 2023-07-14T11:11:42.977 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:42.977 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link delete ceph-brx 2023-07-14T11:11:43.089 INFO:tasks.kclient:client.0 config is {'mntopts': ['ms_mode=crc', 'wsync', 'context=system_u:object_r:postgresql_db_t:s0'], 'syntax': 'v1', 'mountpoint': '/tmp/cephfs'} 2023-07-14T11:11:43.090 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2023-07-14T11:11:43.090 INFO:tasks.cephfs.mount:using client_config["hostfs_mntpt"] = /tmp/cephfs 2023-07-14T11:11:43.090 INFO:tasks.cephfs.mount:hostfs_mntpt = /tmp/cephfs 2023-07-14T11:11:43.091 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph fs ls 2023-07-14T11:11:43.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:43 smithi130 ceph-mon[129405]: pgmap v340: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:43.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:43 smithi040 ceph-mon[97954]: pgmap v340: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:43.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:43 smithi045 ceph-mon[128791]: pgmap v340: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:43.829 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:11:45.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 -- 172.21.15.40:0/434388636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 msgr2=0x7f080410ca20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:45.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 --2- 172.21.15.40:0/434388636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080410ca20 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f07fc00b0a0 tx=0x7f07fc00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 -- 172.21.15.40:0/434388636 shutdown_connections 2023-07-14T11:11:45.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 --2- 172.21.15.40:0/434388636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080410ca20 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 --2- 172.21.15.40:0/434388636 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f08041001d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 --2- 172.21.15.40:0/434388636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 0x7f0804104770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 -- 172.21.15.40:0/434388636 >> 172.21.15.40:0/434388636 conn(0x7f08040fb660 msgr2=0x7f08040fda70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:45.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 -- 172.21.15.40:0/434388636 shutdown_connections 2023-07-14T11:11:45.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.481+0000 7f080b057700 1 -- 172.21.15.40:0/434388636 wait complete. 2023-07-14T11:11:45.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.482+0000 7f080b057700 1 Processor -- start 2023-07-14T11:11:45.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.482+0000 7f080b057700 1 -- start start 2023-07-14T11:11:45.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 0x7f0804198510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080419c980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f080419cf90 con 0x7f08040ffe00 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f080419d100 con 0x7f0804104ea0 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f080b057700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f080419d2c0 con 0x7f08041007a0 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f0808df3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:45.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f0808df3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35374/0 (socket says 172.21.15.40:35374) 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f0808df3700 1 -- 172.21.15.40:0/509572066 learned_addr learned my addr 172.21.15.40:0/509572066 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f08095f4700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080419c980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.483+0000 7f0803fff700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 0x7f0804198510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 -- 172.21.15.40:0/509572066 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 msgr2=0x7f0804198510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 0x7f0804198510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 -- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 msgr2=0x7f0804197fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 -- 172.21.15.40:0/509572066 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07fc009d00 con 0x7f0804104ea0 2023-07-14T11:11:45.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f0808df3700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:45.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f08095f4700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080419c980 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f07fc00b0a0 tx=0x7f07fc00bea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:45.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07fc00e040 con 0x7f0804104ea0 2023-07-14T11:11:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f07fc01d470 con 0x7f0804104ea0 2023-07-14T11:11:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f080419d540 con 0x7f0804104ea0 2023-07-14T11:11:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07fc0052a0 con 0x7f0804104ea0 2023-07-14T11:11:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.484+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08041a4e00 con 0x7f0804104ea0 2023-07-14T11:11:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.485+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f080404e210 con 0x7f0804104ea0 2023-07-14T11:11:45.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.487+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f07fc01d5e0 con 0x7f0804104ea0 2023-07-14T11:11:45.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.489+0000 7f0801ffb700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 0x7f07f40a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:45.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.489+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f07fc0cde70 con 0x7f0804104ea0 2023-07-14T11:11:45.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.489+0000 7f0808df3700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 0x7f07f40a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:45.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.490+0000 7f0808df3700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 0x7f07f40a3680 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f07f8011550 tx=0x7f07f8009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:45.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.491+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f07fc0973c0 con 0x7f0804104ea0 2023-07-14T11:11:45.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:45 smithi040 ceph-mon[97954]: pgmap v341: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:45.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.742+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f0804199330 con 0x7f0804104ea0 2023-07-14T11:11:45.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.743+0000 7f0801ffb700 1 -- 172.21.15.40:0/509572066 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v27) v1 ==== 53+0+88 (secure 0 0 0) 0x7f07fc09a9e0 con 0x7f0804104ea0 2023-07-14T11:11:45.744 INFO:teuthology.orchestra.run.smithi040.stdout:name: cephfs, metadata pool: cephfs_metadata, data pools: [cephfs_data cephfs_data_ec ] 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 msgr2=0x7f07f40a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 0x7f07f40a3680 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f07f8011550 tx=0x7f07f8009260 comp rx=0 tx=0).stop 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 msgr2=0x7f080419c980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080419c980 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f07fc00b0a0 tx=0x7f07fc00bea0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 shutdown_connections 2023-07-14T11:11:45.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f07f40a11d0 0x7f07f40a3680 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0804104ea0 0x7f080419c980 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08040ffe00 0x7f0804197fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 --2- 172.21.15.40:0/509572066 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f08041007a0 0x7f0804198510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 >> 172.21.15.40:0/509572066 conn(0x7f08040fb660 msgr2=0x7f080410af60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 shutdown_connections 2023-07-14T11:11:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:45.745+0000 7f080b057700 1 -- 172.21.15.40:0/509572066 wait complete. 2023-07-14T11:11:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:45 smithi130 ceph-mon[129405]: pgmap v341: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:45.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:45 smithi045 ceph-mon[128791]: pgmap v341: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.7 KiB/s rd, 10 op/s 2023-07-14T11:11:46.210 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2023-07-14T11:11:46.210 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2023-07-14T11:11:46.210 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi040.front.sepia.ceph.com 2023-07-14T11:11:46.210 INFO:tasks.cephfs.mount:self.client.name = client.0 2023-07-14T11:11:46.211 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /tmp/cephfs 2023-07-14T11:11:46.211 INFO:tasks.cephfs.mount:self.cephfs_name = None 2023-07-14T11:11:46.211 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2023-07-14T11:11:46.211 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2023-07-14T11:11:46.211 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/tmp/cephfs' 2023-07-14T11:11:46.212 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.212 DEBUG:teuthology.orchestra.run.smithi040:> ip addr 2023-07-14T11:11:46.257 INFO:teuthology.orchestra.run.smithi040.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2023-07-14T11:11:46.257 INFO:teuthology.orchestra.run.smithi040.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2023-07-14T11:11:46.257 INFO:teuthology.orchestra.run.smithi040.stdout: inet 127.0.0.1/8 scope host lo 2023-07-14T11:11:46.257 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:11:46.257 INFO:teuthology.orchestra.run.smithi040.stdout: inet6 ::1/128 scope host 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:6b:fd:16 brd ff:ff:ff:ff:ff:ff 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp7s0f0 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:6b:fd:17 brd ff:ff:ff:ff:ff:ff 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp7s0f1 2023-07-14T11:11:46.258 INFO:teuthology.orchestra.run.smithi040.stdout:4: ens1f0: mtu 1500 qdisc mq state UP group default qlen 1000 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:bd:12:cc brd ff:ff:ff:ff:ff:ff 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp1s0f0 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: inet 172.21.15.40/20 brd 172.21.15.255 scope global dynamic noprefixroute ens1f0 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft 41395sec preferred_lft 41395sec 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: inet6 fe80::ec4:7aff:febd:12cc/64 scope link 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: valid_lft forever preferred_lft forever 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout:5: ens1f1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2023-07-14T11:11:46.259 INFO:teuthology.orchestra.run.smithi040.stdout: link/ether 0c:c4:7a:bd:12:cd brd ff:ff:ff:ff:ff:ff 2023-07-14T11:11:46.260 INFO:teuthology.orchestra.run.smithi040.stdout: altname enp1s0f1 2023-07-14T11:11:46.260 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2023-07-14T11:11:46.261 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.261 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:11:46.261 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:11:46.261 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link add name ceph-brx type bridge 2023-07-14T11:11:46.262 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip addr flush dev ceph-brx 2023-07-14T11:11:46.262 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set ceph-brx up 2023-07-14T11:11:46.262 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2023-07-14T11:11:46.262 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:11:46.471 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:46 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/509572066' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:11:46.512 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.512 DEBUG:teuthology.orchestra.run.smithi040:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2023-07-14T11:11:46.583 INFO:teuthology.orchestra.run.smithi040.stdout:1 2023-07-14T11:11:46.586 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.586 DEBUG:teuthology.orchestra.run.smithi040:> route 2023-07-14T11:11:46.632 INFO:teuthology.orchestra.run.smithi040.stdout:Kernel IP routing table 2023-07-14T11:11:46.633 INFO:teuthology.orchestra.run.smithi040.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2023-07-14T11:11:46.633 INFO:teuthology.orchestra.run.smithi040.stdout:default _gateway 0.0.0.0 UG 100 0 0 ens1f0 2023-07-14T11:11:46.633 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 ens1f0 2023-07-14T11:11:46.633 INFO:teuthology.orchestra.run.smithi040.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2023-07-14T11:11:46.634 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.634 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:11:46.635 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:11:46.635 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -A FORWARD -o ens1f0 -i ceph-brx -j ACCEPT 2023-07-14T11:11:46.635 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -A FORWARD -i ens1f0 -o ceph-brx -j ACCEPT 2023-07-14T11:11:46.635 DEBUG:teuthology.orchestra.run.smithi040:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o ens1f0 -j MASQUERADE 2023-07-14T11:11:46.635 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:11:46.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:46 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/509572066' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:11:46.812 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.813 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list 2023-07-14T11:11:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:46 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/509572066' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2023-07-14T11:11:46.857 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.858 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list-id 2023-07-14T11:11:46.941 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:46.941 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:11:46.941 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:11:46.942 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns add ceph-ns--tmp-cephfs 2023-07-14T11:11:46.942 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns set ceph-ns--tmp-cephfs 0 2023-07-14T11:11:46.942 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:11:47.128 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--tmp-cephfs' with 192.168.0.1/16 2023-07-14T11:11:47.129 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:47.129 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:11:47.130 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:11:47.130 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link add veth0 netns ceph-ns--tmp-cephfs type veth peer name brx.0 2023-07-14T11:11:47.130 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--tmp-cephfs ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2023-07-14T11:11:47.130 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--tmp-cephfs ip link set veth0 up 2023-07-14T11:11:47.130 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--tmp-cephfs ip link set lo up 2023-07-14T11:11:47.131 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns exec ceph-ns--tmp-cephfs ip route add default via 192.168.255.254 2023-07-14T11:11:47.131 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:11:47.396 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:47.397 DEBUG:teuthology.orchestra.run.smithi040:> (cd / && exec bash -c ' 2023-07-14T11:11:47.397 DEBUG:teuthology.orchestra.run.smithi040:> set -e 2023-07-14T11:11:47.397 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set brx.0 up 2023-07-14T11:11:47.397 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link set dev brx.0 master ceph-brx 2023-07-14T11:11:47.398 DEBUG:teuthology.orchestra.run.smithi040:> ') 2023-07-14T11:11:47.574 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-07-14T11:11:47.574 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -v /tmp/cephfs 2023-07-14T11:11:47.618 INFO:teuthology.orchestra.run.smithi040.stdout:mkdir: created directory '/tmp/cephfs' 2023-07-14T11:11:47.619 INFO:teuthology.orchestra.run:Running command with timeout 60 2023-07-14T11:11:47.619 DEBUG:teuthology.orchestra.run.smithi040:> chmod 0000 /tmp/cephfs 2023-07-14T11:11:47.701 INFO:tasks.cephfs.kernel_mount:mounting using device: :/ 2023-07-14T11:11:47.702 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:47.702 DEBUG:teuthology.orchestra.run.smithi040:> sudo nsenter --net=/var/run/netns/ceph-ns--tmp-cephfs adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/ /tmp/cephfs -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context=system_u:object_r:postgresql_db_t:s0 2023-07-14T11:11:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:47 smithi130 ceph-mon[129405]: pgmap v342: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:11:47.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:47 smithi040 ceph-mon[97954]: pgmap v342: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:11:47.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:47 smithi045 ceph-mon[128791]: pgmap v342: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:11:47.848 INFO:teuthology.orchestra.run.smithi040.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context="system_u:object_r:postgresql_db_t:s0" 2023-07-14T11:11:47.849 INFO:teuthology.orchestra.run.smithi040.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context="system_u:object_r:postgresql_db_t:s0"". 2023-07-14T11:11:47.849 INFO:teuthology.orchestra.run.smithi040.stdout:invalid new device string format 2023-07-14T11:11:48.301 INFO:teuthology.orchestra.run.smithi040.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context="system_u:object_r:postgresql_db_t:s0" 2023-07-14T11:11:48.301 INFO:teuthology.orchestra.run.smithi040.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context="system_u:object_r:postgresql_db_t:s0"". 2023-07-14T11:11:48.301 INFO:teuthology.orchestra.run.smithi040.stdout:invalid new device string format 2023-07-14T11:11:48.301 INFO:teuthology.orchestra.run.smithi040.stdout:mount.ceph: resolved to: "172.21.15.40:3300,172.21.15.45:3300,172.21.15.130:3300" 2023-07-14T11:11:48.302 INFO:teuthology.orchestra.run.smithi040.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.40:3300,172.21.15.45:3300,172.21.15.130:3300:/ 2023-07-14T11:11:48.302 INFO:teuthology.orchestra.run.smithi040.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,ms_mode=crc,wsync,context="system_u:object_r:postgresql_db_t:s0",key=0,fsid=bbe0b002-2235-11ee-9b32-001a4aab830c" will pass to kernel 2023-07-14T11:11:48.306 INFO:tasks.cephfs.kernel_mount:mount command passed 2023-07-14T11:11:48.307 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:48.307 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 1777 /tmp/cephfs 2023-07-14T11:11:48.378 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:48.379 DEBUG:teuthology.orchestra.run.smithi040:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2023-07-14T11:11:48.466 INFO:teuthology.orchestra.run.smithi040.stdout:client15396 2023-07-14T11:11:48.467 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:11:48.467 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:11:48.510 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:48.511 DEBUG:teuthology.orchestra.run.smithi040:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2023-07-14T11:11:48.637 INFO:teuthology.orchestra.run.smithi040.stdout:client15396 2023-07-14T11:11:48.638 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:48.638 DEBUG:teuthology.orchestra.run.smithi040:> (cd /tmp/cephfs && exec bash -c 'getfattr --only-values -n ceph.client_id .') 2023-07-14T11:11:48.724 INFO:teuthology.orchestra.run.smithi040.stdout:client15396 2023-07-14T11:11:48.725 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:11:48.725 DEBUG:teuthology.orchestra.run.smithi040:> (cd /tmp/cephfs && exec bash -c 'sudo dd if=/sys/kernel/debug/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c.client15396/status') 2023-07-14T11:11:48.835 INFO:teuthology.orchestra.run.smithi040.stdout:instance: client.15396 (3)192.168.0.1:0/1526157301 2023-07-14T11:11:48.835 INFO:teuthology.orchestra.run.smithi040.stdout:blocklisted: false 2023-07-14T11:11:48.836 INFO:teuthology.orchestra.run.smithi040.stderr:0+1 records in 2023-07-14T11:11:48.836 INFO:teuthology.orchestra.run.smithi040.stderr:0+1 records out 2023-07-14T11:11:48.836 INFO:teuthology.orchestra.run.smithi040.stderr:70 bytes copied, 5.6386e-05 s, 1.2 MB/s 2023-07-14T11:11:48.838 INFO:teuthology.run_tasks:Running task check-counter... 2023-07-14T11:11:48.849 INFO:teuthology.run_tasks:Running task fwd_scrub... 2023-07-14T11:11:48.887 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:49.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.090+0000 7f1209bff700 1 -- 172.21.15.40:0/1367962749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 msgr2=0x7f1204107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.090+0000 7f1209bff700 1 --2- 172.21.15.40:0/1367962749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f1204107f40 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f11fc00b0a0 tx=0x7f11fc00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 -- 172.21.15.40:0/1367962749 shutdown_connections 2023-07-14T11:11:49.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 --2- 172.21.15.40:0/1367962749 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f1204107f40 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 --2- 172.21.15.40:0/1367962749 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f12040ffb00 0x7f12040fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 --2- 172.21.15.40:0/1367962749 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1204100cf0 0x7f1204101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 -- 172.21.15.40:0/1367962749 >> 172.21.15.40:0/1367962749 conn(0x7f12040fb0e0 msgr2=0x7f12040fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:49.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 -- 172.21.15.40:0/1367962749 shutdown_connections 2023-07-14T11:11:49.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.091+0000 7f1209bff700 1 -- 172.21.15.40:0/1367962749 wait complete. 2023-07-14T11:11:49.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 Processor -- start 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 -- start start 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 0x7f1204197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1204100cf0 0x7f1204198080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f120419f6c0 con 0x7f1204101680 2023-07-14T11:11:49.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f120419f800 con 0x7f12040ffb00 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1209bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f120419f990 con 0x7f1204100cf0 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.092+0000 7f1203fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f12037fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 0x7f1204197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35406/0 (socket says 172.21.15.40:35406) 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 -- 172.21.15.40:0/3048806403 learned_addr learned my addr 172.21.15.40:0/3048806403 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1202ffd700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1204100cf0 0x7f1204198080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 -- 172.21.15.40:0/3048806403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1204100cf0 msgr2=0x7f1204198080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1204100cf0 0x7f1204198080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 -- 172.21.15.40:0/3048806403 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 msgr2=0x7f1204197b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 0x7f1204197b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 -- 172.21.15.40:0/3048806403 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11fc009d00 con 0x7f1204101680 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f12037fe700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 0x7f1204197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1203fff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f11fc00d010 tx=0x7f11fc00bc80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11fc00e040 con 0x7f1204101680 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11fc003e10 con 0x7f1204101680 2023-07-14T11:11:49.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11fc005260 con 0x7f1204101680 2023-07-14T11:11:49.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.093+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f120419fc10 con 0x7f1204101680 2023-07-14T11:11:49.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.094+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f120419ffe0 con 0x7f1204101680 2023-07-14T11:11:49.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.095+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f120410b3a0 con 0x7f1204101680 2023-07-14T11:11:49.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.096+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f11fc0053e0 con 0x7f1204101680 2023-07-14T11:11:49.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.098+0000 7f1200ff9700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 0x7f11ec0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.098+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f11fc0ce0d0 con 0x7f1204101680 2023-07-14T11:11:49.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.098+0000 7f12037fe700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 0x7f11ec0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.099+0000 7f12037fe700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 0x7f11ec0a36d0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f11f4005950 tx=0x7f11f40058e0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:49.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.102+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f11fc097500 con 0x7f1204101680 2023-07-14T11:11:49.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.311+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f12041a0590 con 0x7f1204101680 2023-07-14T11:11:49.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.313+0000 7f1200ff9700 1 -- 172.21.15.40:0/3048806403 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f11fc09ab20 con 0x7f1204101680 2023-07-14T11:11:49.314 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:49.315 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:49.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 msgr2=0x7f11ec0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 0x7f11ec0a36d0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f11f4005950 tx=0x7f11f40058e0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 msgr2=0x7f120419f0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f11fc00d010 tx=0x7f11fc00bc80 comp rx=0 tx=0).stop 2023-07-14T11:11:49.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 shutdown_connections 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f11ec0a1220 0x7f11ec0a36d0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1204101680 0x7f120419f0f0 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f12040ffb00 0x7f1204197b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 --2- 172.21.15.40:0/3048806403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1204100cf0 0x7f1204198080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 >> 172.21.15.40:0/3048806403 conn(0x7f12040fb0e0 msgr2=0x7f12041064d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 shutdown_connections 2023-07-14T11:11:49.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.315+0000 7f1209bff700 1 -- 172.21.15.40:0/3048806403 wait complete. 2023-07-14T11:11:49.336 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:49.337 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:49.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 -- 172.21.15.40:0/1354943885 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4100cd0 msgr2=0x7f53c4101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 --2- 172.21.15.40:0/1354943885 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4100cd0 0x7f53c4101120 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f53b8009990 tx=0x7f53b8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 -- 172.21.15.40:0/1354943885 shutdown_connections 2023-07-14T11:11:49.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 --2- 172.21.15.40:0/1354943885 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c4101660 0x7f53c4107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 --2- 172.21.15.40:0/1354943885 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c40ffae0 0x7f53c40ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 --2- 172.21.15.40:0/1354943885 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4100cd0 0x7f53c4101120 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 -- 172.21.15.40:0/1354943885 >> 172.21.15.40:0/1354943885 conn(0x7f53c40fb0e0 msgr2=0x7f53c40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:49.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 -- 172.21.15.40:0/1354943885 shutdown_connections 2023-07-14T11:11:49.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.544+0000 7f53c9fff700 1 -- 172.21.15.40:0/1354943885 wait complete. 2023-07-14T11:11:49.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 Processor -- start 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 -- start start 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c4100cd0 0x7f53c4195d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 0x7f53c419cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53c419d3a0 con 0x7f53c40ffae0 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53c419d4e0 con 0x7f53c4101660 2023-07-14T11:11:49.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.546+0000 7f53c9fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53c419d670 con 0x7f53c4100cd0 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35418/0 (socket says 172.21.15.40:35418) 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 -- 172.21.15.40:0/532539698 learned_addr learned my addr 172.21.15.40:0/532539698 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c2ffd700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c4100cd0 0x7f53c4195d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c3fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 0x7f53c419cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 -- 172.21.15.40:0/532539698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c4100cd0 msgr2=0x7f53c4195d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c4100cd0 0x7f53c4195d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 -- 172.21.15.40:0/532539698 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 msgr2=0x7f53c419cdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 0x7f53c419cdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 -- 172.21.15.40:0/532539698 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53b80095f0 con 0x7f53c40ffae0 2023-07-14T11:11:49.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c3fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 0x7f53c419cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.547+0000 7f53c37fe700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f53b400e970 tx=0x7f53b400ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.548+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53b4009790 con 0x7f53c40ffae0 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.548+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f53c419d950 con 0x7f53c40ffae0 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.548+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f53b400c7e0 con 0x7f53c40ffae0 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.548+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53b4003c70 con 0x7f53c40ffae0 2023-07-14T11:11:49.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.548+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53c419deb0 con 0x7f53c40ffae0 2023-07-14T11:11:49.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.549+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f53c410b2b0 con 0x7f53c40ffae0 2023-07-14T11:11:49.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.551+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f53b400cc50 con 0x7f53c40ffae0 2023-07-14T11:11:49.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.552+0000 7f53c0ff9700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 0x7f53ac0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.552+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f53b4014070 con 0x7f53c40ffae0 2023-07-14T11:11:49.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.552+0000 7f53c2ffd700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 0x7f53ac0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.553+0000 7f53c2ffd700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 0x7f53ac0a3600 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f53b800b620 tx=0x7f53b8005960 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:49.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.556+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f53b4096210 con 0x7f53c40ffae0 2023-07-14T11:11:49.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.765+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f53c4106a20 con 0x7f53c40ffae0 2023-07-14T11:11:49.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.765+0000 7f53c0ff9700 1 -- 172.21.15.40:0/532539698 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7f53b4003dd0 con 0x7f53c40ffae0 2023-07-14T11:11:49.767 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:49.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.767+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 msgr2=0x7f53ac0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.767+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 0x7f53ac0a3600 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f53b800b620 tx=0x7f53b8005960 comp rx=0 tx=0).stop 2023-07-14T11:11:49.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.767+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 msgr2=0x7f53c4195820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.767+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f53b400e970 tx=0x7f53b400ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 shutdown_connections 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53ac0a1150 0x7f53ac0a3600 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53c4101660 0x7f53c419cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53c40ffae0 0x7f53c4195820 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 --2- 172.21.15.40:0/532539698 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53c4100cd0 0x7f53c4195d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 >> 172.21.15.40:0/532539698 conn(0x7f53c40fb0e0 msgr2=0x7f53c40fd3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:49.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 shutdown_connections 2023-07-14T11:11:49.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.768+0000 7f53c9fff700 1 -- 172.21.15.40:0/532539698 wait complete. 2023-07-14T11:11:49.787 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:49.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 -- 172.21.15.40:0/3842663137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0101660 msgr2=0x7fcaf0107f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/3842663137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0101660 0x7fcaf0107f20 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7fcae800b0a0 tx=0x7fcae800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 -- 172.21.15.40:0/3842663137 shutdown_connections 2023-07-14T11:11:49.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/3842663137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0101660 0x7fcaf0107f20 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/3842663137 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf00ffae0 0x7fcaf00ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/3842663137 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf0100cd0 0x7fcaf0101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 -- 172.21.15.40:0/3842663137 >> 172.21.15.40:0/3842663137 conn(0x7fcaf00fb0e0 msgr2=0x7fcaf00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:49.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 -- 172.21.15.40:0/3842663137 shutdown_connections 2023-07-14T11:11:49.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.993+0000 7fcaf59ff700 1 -- 172.21.15.40:0/3842663137 wait complete. 2023-07-14T11:11:49.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.994+0000 7fcaf59ff700 1 Processor -- start 2023-07-14T11:11:49.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 -- start start 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0100cd0 0x7fcaf01980a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf0101660 0x7fcaf019f110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcaf019f6e0 con 0x7fcaf0100cd0 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcaf019f820 con 0x7fcaf00ffae0 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaf59ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcaf019f9b0 con 0x7fcaf0101660 2023-07-14T11:11:49.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaeeffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaee7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0100cd0 0x7fcaf01980a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaeeffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59592/0 (socket says 172.21.15.40:59592) 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaeeffd700 1 -- 172.21.15.40:0/1449593406 learned_addr learned my addr 172.21.15.40:0/1449593406 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.995+0000 7fcaef7fe700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf0101660 0x7fcaf019f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 -- 172.21.15.40:0/1449593406 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf0101660 msgr2=0x7fcaf019f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf0101660 0x7fcaf019f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 -- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0100cd0 msgr2=0x7fcaf01980a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:49.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0100cd0 0x7fcaf01980a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 -- 172.21.15.40:0/1449593406 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcae0009520 con 0x7fcaf00ffae0 2023-07-14T11:11:50.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaeeffd700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fcae000ea50 tx=0x7fcae000ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcae000cbd0 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcae8009d00 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcae000cd30 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcae0018680 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.996+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcaf01a0060 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:49.997+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcaf010b310 con 0x7fcaf00ffae0 2023-07-14T11:11:50.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.000+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fcae0010430 con 0x7fcaf00ffae0 2023-07-14T11:11:50.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.001+0000 7fcaf49fd700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 0x7fcad80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.001+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fcae0014070 con 0x7fcaf00ffae0 2023-07-14T11:11:50.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.001+0000 7fcaee7fc700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 0x7fcad80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.002+0000 7fcaee7fc700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 0x7fcad80a3600 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fcae4009c60 tx=0x7fcae4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.003+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcae0096b00 con 0x7fcaf00ffae0 2023-07-14T11:11:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:49 smithi130 ceph-mon[129405]: pgmap v343: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:49 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3048806403' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:49 smithi040 ceph-mon[97954]: pgmap v343: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:49 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3048806403' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:50.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:49 smithi045 ceph-mon[128791]: pgmap v343: 129 pgs: 129 active+clean; 586 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.5 KiB/s rd, 9 op/s 2023-07-14T11:11:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:49 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3048806403' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:50.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.212+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fcaf01a0610 con 0x7fcaf00ffae0 2023-07-14T11:11:50.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.213+0000 7fcaf49fd700 1 -- 172.21.15.40:0/1449593406 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7fcae009a120 con 0x7fcaf00ffae0 2023-07-14T11:11:50.215 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:50.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.215+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 msgr2=0x7fcad80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.215+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 0x7fcad80a3600 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fcae4009c60 tx=0x7fcae4009260 comp rx=0 tx=0).stop 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.215+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 msgr2=0x7fcaf0197b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.215+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fcae000ea50 tx=0x7fcae000ed60 comp rx=0 tx=0).stop 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 shutdown_connections 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcad80a1150 0x7fcad80a3600 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcaf0101660 0x7fcaf019f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcaf00ffae0 0x7fcaf0197b60 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 --2- 172.21.15.40:0/1449593406 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcaf0100cd0 0x7fcaf01980a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 >> 172.21.15.40:0/1449593406 conn(0x7fcaf00fb0e0 msgr2=0x7fcaf0106500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:50.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 shutdown_connections 2023-07-14T11:11:50.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.216+0000 7fcaf59ff700 1 -- 172.21.15.40:0/1449593406 wait complete. 2023-07-14T11:11:50.236 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:50.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.441+0000 7fea9b1ff700 1 -- 172.21.15.40:0/3818954329 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 msgr2=0x7fea94101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.441+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/3818954329 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94101120 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7fea80009990 tx=0x7fea80009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 -- 172.21.15.40:0/3818954329 shutdown_connections 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/3818954329 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea94101660 0x7fea94107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/3818954329 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea940ffae0 0x7fea940ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/3818954329 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94101120 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 -- 172.21.15.40:0/3818954329 >> 172.21.15.40:0/3818954329 conn(0x7fea940fb0e0 msgr2=0x7fea940fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:50.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 -- 172.21.15.40:0/3818954329 shutdown_connections 2023-07-14T11:11:50.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.442+0000 7fea9b1ff700 1 -- 172.21.15.40:0/3818954329 wait complete. 2023-07-14T11:11:50.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 Processor -- start 2023-07-14T11:11:50.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 -- start start 2023-07-14T11:11:50.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea940ffae0 0x7fea94197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea94101660 0x7fea9419f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9419f660 con 0x7fea94100cd0 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9419f7a0 con 0x7fea940ffae0 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea9b1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea9419f930 con 0x7fea94101660 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea93fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea93fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35470/0 (socket says 172.21.15.40:35470) 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea93fff700 1 -- 172.21.15.40:0/1933816287 learned_addr learned my addr 172.21.15.40:0/1933816287 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.443+0000 7fea98f9b700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea940ffae0 0x7fea94197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea9979c700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea94101660 0x7fea9419f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 -- 172.21.15.40:0/1933816287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea94101660 msgr2=0x7fea9419f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea94101660 0x7fea9419f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 -- 172.21.15.40:0/1933816287 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea940ffae0 msgr2=0x7fea94197ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea940ffae0 0x7fea94197ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 -- 172.21.15.40:0/1933816287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea800095f0 con 0x7fea94100cd0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea93fff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fea8000b620 tx=0x7fea80005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea8001d070 con 0x7fea94100cd0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea80022470 con 0x7fea94100cd0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea9419fbb0 con 0x7fea94100cd0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.444+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea80017650 con 0x7fea94100cd0 2023-07-14T11:11:50.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.445+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea941a0030 con 0x7fea94100cd0 2023-07-14T11:11:50.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.446+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea941921c0 con 0x7fea94100cd0 2023-07-14T11:11:50.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.448+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fea800225e0 con 0x7fea94100cd0 2023-07-14T11:11:50.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.449+0000 7fea91ffb700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 0x7fea840a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.449+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fea800ce0b0 con 0x7fea94100cd0 2023-07-14T11:11:50.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.450+0000 7fea98f9b700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 0x7fea840a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.450+0000 7fea98f9b700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 0x7fea840a3680 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fea94100b30 tx=0x7fea88009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.453+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fea800974e0 con 0x7fea94100cd0 2023-07-14T11:11:50.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.661+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fea940611c0 con 0x7fea94100cd0 2023-07-14T11:11:50.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.662+0000 7fea91ffb700 1 -- 172.21.15.40:0/1933816287 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7fea80027020 con 0x7fea94100cd0 2023-07-14T11:11:50.664 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:50.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.664+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 msgr2=0x7fea840a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.664+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 0x7fea840a3680 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fea94100b30 tx=0x7fea88009260 comp rx=0 tx=0).stop 2023-07-14T11:11:50.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.664+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 msgr2=0x7fea94198020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.664+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7fea8000b620 tx=0x7fea80005740 comp rx=0 tx=0).stop 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 shutdown_connections 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea840a11d0 0x7fea840a3680 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea94101660 0x7fea9419f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea940ffae0 0x7fea94197ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 --2- 172.21.15.40:0/1933816287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea94100cd0 0x7fea94198020 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 >> 172.21.15.40:0/1933816287 conn(0x7fea940fb0e0 msgr2=0x7fea941065a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:50.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 shutdown_connections 2023-07-14T11:11:50.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.665+0000 7fea9b1ff700 1 -- 172.21.15.40:0/1933816287 wait complete. 2023-07-14T11:11:50.685 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:50.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 -- 172.21.15.40:0/1759053379 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b50100cd0 msgr2=0x7f4b50101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 --2- 172.21.15.40:0/1759053379 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b50100cd0 0x7f4b50101120 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f4b44009990 tx=0x7f4b44009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 -- 172.21.15.40:0/1759053379 shutdown_connections 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 --2- 172.21.15.40:0/1759053379 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b50107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 --2- 172.21.15.40:0/1759053379 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b500ffae0 0x7f4b500ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 --2- 172.21.15.40:0/1759053379 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b50100cd0 0x7f4b50101120 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 -- 172.21.15.40:0/1759053379 >> 172.21.15.40:0/1759053379 conn(0x7f4b500fb0e0 msgr2=0x7f4b500fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 -- 172.21.15.40:0/1759053379 shutdown_connections 2023-07-14T11:11:50.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.892+0000 7f4b557ff700 1 -- 172.21.15.40:0/1759053379 wait complete. 2023-07-14T11:11:50.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.893+0000 7f4b557ff700 1 Processor -- start 2023-07-14T11:11:50.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 -- start start 2023-07-14T11:11:50.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b50100cd0 0x7f4b50198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b5019f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5019f660 con 0x7f4b500ffae0 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5019f7a0 con 0x7f4b50100cd0 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b557ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b5019f930 con 0x7f4b50101660 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b4effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b4effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35494/0 (socket says 172.21.15.40:35494) 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b4effd700 1 -- 172.21.15.40:0/3573251656 learned_addr learned my addr 172.21.15.40:0/3573251656 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:50.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b4e7fc700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b50100cd0 0x7f4b50198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.894+0000 7f4b4f7fe700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b5019f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 -- 172.21.15.40:0/3573251656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 msgr2=0x7f4b5019f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b5019f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 -- 172.21.15.40:0/3573251656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b50100cd0 msgr2=0x7f4b50198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b50100cd0 0x7f4b50198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 -- 172.21.15.40:0/3573251656 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b40009520 con 0x7f4b500ffae0 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4f7fe700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b5019f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:50.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b4effd700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f4b4000ea50 tx=0x7f4b4000ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b4000cbd0 con 0x7f4b500ffae0 2023-07-14T11:11:50.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b440095f0 con 0x7f4b500ffae0 2023-07-14T11:11:50.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b4000cd30 con 0x7f4b500ffae0 2023-07-14T11:11:50.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b40018680 con 0x7f4b500ffae0 2023-07-14T11:11:50.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.895+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b5019ffe0 con 0x7f4b500ffae0 2023-07-14T11:11:50.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.896+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b501921c0 con 0x7f4b500ffae0 2023-07-14T11:11:50.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.899+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f4b400188a0 con 0x7f4b500ffae0 2023-07-14T11:11:50.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.900+0000 7f4b37fff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 0x7f4b380a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:50.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.900+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f4b40014070 con 0x7f4b500ffae0 2023-07-14T11:11:50.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.900+0000 7f4b4e7fc700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 0x7f4b380a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:50.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.901+0000 7f4b4e7fc700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 0x7f4b380a3600 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f4b4400b2e0 tx=0x7f4b44005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:50.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:50.901+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4b40096b70 con 0x7f4b500ffae0 2023-07-14T11:11:51.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/532539698' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1449593406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:50 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1933816287' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/532539698' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1449593406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:50 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1933816287' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/532539698' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1449593406' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:50 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1933816287' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:51.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.114+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f4b5002cb00 con 0x7f4b500ffae0 2023-07-14T11:11:51.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.114+0000 7f4b37fff700 1 -- 172.21.15.40:0/3573251656 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7f4b400b8e60 con 0x7f4b500ffae0 2023-07-14T11:11:51.116 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:51.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.116+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 msgr2=0x7f4b380a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.116+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 0x7f4b380a3600 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f4b4400b2e0 tx=0x7f4b44005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.116+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 msgr2=0x7f4b50197ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.116+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f4b4000ea50 tx=0x7f4b4000ed60 comp rx=0 tx=0).stop 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 shutdown_connections 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f4b380a1150 0x7f4b380a3600 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b50101660 0x7f4b5019f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b500ffae0 0x7f4b50197ae0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 --2- 172.21.15.40:0/3573251656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b50100cd0 0x7f4b50198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 >> 172.21.15.40:0/3573251656 conn(0x7f4b500fb0e0 msgr2=0x7f4b501065a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:51.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 shutdown_connections 2023-07-14T11:11:51.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.117+0000 7f4b557ff700 1 -- 172.21.15.40:0/3573251656 wait complete. 2023-07-14T11:11:51.137 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:51.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 -- 172.21.15.40:0/621678365 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88800ffb00 msgr2=0x7f88800fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 --2- 172.21.15.40:0/621678365 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88800ffb00 0x7f88800fff10 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f887c009990 tx=0x7f887c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 -- 172.21.15.40:0/621678365 shutdown_connections 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 --2- 172.21.15.40:0/621678365 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 0x7f8880107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 --2- 172.21.15.40:0/621678365 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88800ffb00 0x7f88800fff10 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 --2- 172.21.15.40:0/621678365 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8880100cf0 0x7f8880101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.338+0000 7f88881ff700 1 -- 172.21.15.40:0/621678365 >> 172.21.15.40:0/621678365 conn(0x7f88800fb0e0 msgr2=0x7f88800fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.339+0000 7f88881ff700 1 -- 172.21.15.40:0/621678365 shutdown_connections 2023-07-14T11:11:51.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.339+0000 7f88881ff700 1 -- 172.21.15.40:0/621678365 wait complete. 2023-07-14T11:11:51.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.339+0000 7f88881ff700 1 Processor -- start 2023-07-14T11:11:51.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 -- start start 2023-07-14T11:11:51.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f88800ffb00 0x7f88800742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 0x7f8880072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88800735e0 con 0x7f8880100cf0 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8880073750 con 0x7f88800ffb00 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f88881ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f888019f860 con 0x7f8880101680 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f888579a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f888579a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35514/0 (socket says 172.21.15.40:35514) 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f888579a700 1 -- 172.21.15.40:0/2191916174 learned_addr learned my addr 172.21.15.40:0/2191916174 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:51.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f8885f9b700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f88800ffb00 0x7f88800742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.340+0000 7f888679c700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 0x7f8880072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 -- 172.21.15.40:0/2191916174 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 msgr2=0x7f8880072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 0x7f8880072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 -- 172.21.15.40:0/2191916174 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f88800ffb00 msgr2=0x7f88800742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f88800ffb00 0x7f88800742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 -- 172.21.15.40:0/2191916174 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f887c0095f0 con 0x7f8880100cf0 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f888579a700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f887000e9a0 tx=0x7f887000ecb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:51.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8870009790 con 0x7f8880100cf0 2023-07-14T11:11:51.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8870010430 con 0x7f8880100cf0 2023-07-14T11:11:51.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8870018680 con 0x7f8880100cf0 2023-07-14T11:11:51.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f888019fb40 con 0x7f8880100cf0 2023-07-14T11:11:51.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.341+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f88801a0050 con 0x7f8880100cf0 2023-07-14T11:11:51.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.342+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f888010b320 con 0x7f8880100cf0 2023-07-14T11:11:51.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.344+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f88700105a0 con 0x7f8880100cf0 2023-07-14T11:11:51.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.346+0000 7f8876ffd700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 0x7f886c0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.346+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f8870014070 con 0x7f8880100cf0 2023-07-14T11:11:51.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.346+0000 7f8885f9b700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 0x7f886c0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.347+0000 7f8885f9b700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 0x7f886c0a36d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f887c00b2e0 tx=0x7f887c005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:51.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.350+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8870051790 con 0x7f8880100cf0 2023-07-14T11:11:51.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.558+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f888010b510 con 0x7f8880100cf0 2023-07-14T11:11:51.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.560+0000 7f8876ffd700 1 -- 172.21.15.40:0/2191916174 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f88700961b0 con 0x7f8880100cf0 2023-07-14T11:11:51.562 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:51.562 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:51.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 msgr2=0x7f886c0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 0x7f886c0a36d0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f887c00b2e0 tx=0x7f887c005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 msgr2=0x7f8880072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f887000e9a0 tx=0x7f887000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 shutdown_connections 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f886c0a1220 0x7f886c0a36d0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8880101680 0x7f8880072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f88800ffb00 0x7f88800742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 --2- 172.21.15.40:0/2191916174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8880100cf0 0x7f8880072900 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.562+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 >> 172.21.15.40:0/2191916174 conn(0x7f88800fb0e0 msgr2=0x7f88800fd300 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:51.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.563+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 shutdown_connections 2023-07-14T11:11:51.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.563+0000 7f88881ff700 1 -- 172.21.15.40:0/2191916174 wait complete. 2023-07-14T11:11:51.583 INFO:tasks.fwd_scrub.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:51.584 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:51.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.793+0000 7f1bb0fff700 1 -- 172.21.15.40:0/264939687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac072f70 msgr2=0x7f1bac0733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.793+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/264939687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac072f70 0x7f1bac0733e0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f1b940098f0 tx=0x7f1b94009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 -- 172.21.15.40:0/264939687 shutdown_connections 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/264939687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac1079a0 0x7f1bac109e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/264939687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 0x7f1bac0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/264939687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac072f70 0x7f1bac0733e0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 -- 172.21.15.40:0/264939687 >> 172.21.15.40:0/264939687 conn(0x7f1bac0fb0e0 msgr2=0x7f1bac0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:51.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 -- 172.21.15.40:0/264939687 shutdown_connections 2023-07-14T11:11:51.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.794+0000 7f1bb0fff700 1 -- 172.21.15.40:0/264939687 wait complete. 2023-07-14T11:11:51.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 Processor -- start 2023-07-14T11:11:51.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 -- start start 2023-07-14T11:11:51.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac072f70 0x7f1bac197b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 0x7f1bac198060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bac19f6a0 con 0x7f1bac1079a0 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bac19f7e0 con 0x7f1bac074550 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1bb0fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1bac19f970 con 0x7f1bac072f70 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1baad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1baad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35540/0 (socket says 172.21.15.40:35540) 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1baad9d700 1 -- 172.21.15.40:0/52085767 learned_addr learned my addr 172.21.15.40:0/52085767 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1baa59c700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac072f70 0x7f1bac197b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.795+0000 7f1ba3fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 0x7f1bac198060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 -- 172.21.15.40:0/52085767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac072f70 msgr2=0x7f1bac197b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac072f70 0x7f1bac197b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 -- 172.21.15.40:0/52085767 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 msgr2=0x7f1bac198060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 0x7f1bac198060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:51.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 -- 172.21.15.40:0/52085767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1b9c0095f0 con 0x7f1bac1079a0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1baad9d700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1ba400ef30 tx=0x7f1ba400c3a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ba4018070 con 0x7f1bac1079a0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ba400f040 con 0x7f1bac1079a0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1b94009520 con 0x7f1bac1079a0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.796+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ba400cd00 con 0x7f1bac1079a0 2023-07-14T11:11:51.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.797+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1bac19ff70 con 0x7f1bac1079a0 2023-07-14T11:11:51.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.798+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1bac106050 con 0x7f1bac1079a0 2023-07-14T11:11:51.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.800+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1ba4003a00 con 0x7f1bac1079a0 2023-07-14T11:11:51.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.801+0000 7f1ba37fe700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 0x7f1b980a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:51.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.801+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f1ba40cc770 con 0x7f1bac1079a0 2023-07-14T11:11:51.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.801+0000 7f1baa59c700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 0x7f1b980a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:51.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.802+0000 7f1baa59c700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 0x7f1b980a3680 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1bac074920 tx=0x7f1b9c009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:51.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:51.805+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1ba4095ba0 con 0x7f1bac1079a0 2023-07-14T11:11:52.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.012+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f1bac0611c0 con 0x7f1bac1079a0 2023-07-14T11:11:52.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.013+0000 7f1ba37fe700 1 -- 172.21.15.40:0/52085767 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7f1ba40991c0 con 0x7f1bac1079a0 2023-07-14T11:11:52.015 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:52.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 msgr2=0x7f1b980a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 0x7f1b980a3680 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1bac074920 tx=0x7f1b9c009310 comp rx=0 tx=0).stop 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 msgr2=0x7f1bac19f0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7f1ba400ef30 tx=0x7f1ba400c3a0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 shutdown_connections 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1b980a11d0 0x7f1b980a3680 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1bac1079a0 0x7f1bac19f0d0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1bac072f70 0x7f1bac197b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 --2- 172.21.15.40:0/52085767 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1bac074550 0x7f1bac198060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 >> 172.21.15.40:0/52085767 conn(0x7f1bac0fb0e0 msgr2=0x7f1bac0fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:52.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 shutdown_connections 2023-07-14T11:11:52.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.015+0000 7f1bb0fff700 1 -- 172.21.15.40:0/52085767 wait complete. 2023-07-14T11:11:52.033 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:51 smithi040 ceph-mon[97954]: pgmap v344: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:52.033 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:51 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3573251656' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:52.033 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:51 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2191916174' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:52.035 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:51 smithi130 ceph-mon[129405]: pgmap v344: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:51 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3573251656' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:51 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2191916174' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:51 smithi045 ceph-mon[128791]: pgmap v344: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.8 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:51 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3573251656' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:51 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2191916174' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:52.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 -- 172.21.15.40:0/3320779026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c0ffae0 msgr2=0x7fc80c0ffef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3320779026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c0ffae0 0x7fc80c0ffef0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc0098f0 tx=0x7fc7fc009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:52.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 -- 172.21.15.40:0/3320779026 shutdown_connections 2023-07-14T11:11:52.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3320779026 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 0x7fc80c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3320779026 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c0ffae0 0x7fc80c0ffef0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3320779026 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c100cd0 0x7fc80c101120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 -- 172.21.15.40:0/3320779026 >> 172.21.15.40:0/3320779026 conn(0x7fc80c0fb0e0 msgr2=0x7fc80c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:52.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 -- 172.21.15.40:0/3320779026 shutdown_connections 2023-07-14T11:11:52.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.241+0000 7fc8117ff700 1 -- 172.21.15.40:0/3320779026 wait complete. 2023-07-14T11:11:52.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.242+0000 7fc8117ff700 1 Processor -- start 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.242+0000 7fc8117ff700 1 -- start start 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.242+0000 7fc8117ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc8117ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 0x7fc80c197f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc8117ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 0x7fc80c19eff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc8117ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc80c19f5c0 con 0x7fc80c101660 2023-07-14T11:11:52.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc8117ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc80c19f700 con 0x7fc80c0ffae0 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc8117ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc80c19f890 con 0x7fc80c100cd0 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59668/0 (socket says 172.21.15.40:59668) 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 -- 172.21.15.40:0/3891186161 learned_addr learned my addr 172.21.15.40:0/3891186161 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80b7fe700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 0x7fc80c19eff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc803fff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 0x7fc80c197f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 -- 172.21.15.40:0/3891186161 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 msgr2=0x7fc80c197f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 0x7fc80c197f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 -- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 msgr2=0x7fc80c19eff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 0x7fc80c19eff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 -- 172.21.15.40:0/3891186161 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc804009d00 con 0x7fc80c0ffae0 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc803fff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 0x7fc80c197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.243+0000 7fc80affd700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc000c00 tx=0x7fc7fc00b790 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.244+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7fc01c070 con 0x7fc80c0ffae0 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.244+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc7fc009520 con 0x7fc80c0ffae0 2023-07-14T11:11:52.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.244+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc7fc00f7d0 con 0x7fc80c0ffae0 2023-07-14T11:11:52.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.244+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc7fc003ae0 con 0x7fc80c0ffae0 2023-07-14T11:11:52.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.244+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc80c19fe50 con 0x7fc80c0ffae0 2023-07-14T11:11:52.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.245+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc80c10b250 con 0x7fc80c0ffae0 2023-07-14T11:11:52.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.247+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc7fc020030 con 0x7fc80c0ffae0 2023-07-14T11:11:52.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.248+0000 7fc808ff9700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 0x7fc7f80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.249+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc7fc0ce120 con 0x7fc80c0ffae0 2023-07-14T11:11:52.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.249+0000 7fc803fff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 0x7fc7f80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.249+0000 7fc803fff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 0x7fc7f80a3600 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fc7f4009cb0 tx=0x7fc7f4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:52.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.251+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc7fc097550 con 0x7fc80c0ffae0 2023-07-14T11:11:52.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.460+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fc80c10b3e0 con 0x7fc80c0ffae0 2023-07-14T11:11:52.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.461+0000 7fc808ff9700 1 -- 172.21.15.40:0/3891186161 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7fc7fc09ab70 con 0x7fc80c0ffae0 2023-07-14T11:11:52.463 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 msgr2=0x7fc7f80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 0x7fc7f80a3600 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fc7f4009cb0 tx=0x7fc7f4009260 comp rx=0 tx=0).stop 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 msgr2=0x7fc80c197a40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fc7fc000c00 tx=0x7fc7fc00b790 comp rx=0 tx=0).stop 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 shutdown_connections 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc7f80a1150 0x7fc7f80a3600 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc80c101660 0x7fc80c19eff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc80c0ffae0 0x7fc80c197a40 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 --2- 172.21.15.40:0/3891186161 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc80c100cd0 0x7fc80c197f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 >> 172.21.15.40:0/3891186161 conn(0x7fc80c0fb0e0 msgr2=0x7fc80c0fd3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:52.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 shutdown_connections 2023-07-14T11:11:52.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.464+0000 7fc8117ff700 1 -- 172.21.15.40:0/3891186161 wait complete. 2023-07-14T11:11:52.484 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:52.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.691+0000 7fce915ff700 1 -- 172.21.15.40:0/3354375696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c100cd0 msgr2=0x7fce8c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.691+0000 7fce915ff700 1 --2- 172.21.15.40:0/3354375696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c100cd0 0x7fce8c101120 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 crypto rx=0x7fce80009990 tx=0x7fce80009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 -- 172.21.15.40:0/3354375696 shutdown_connections 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 --2- 172.21.15.40:0/3354375696 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 0x7fce8c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 --2- 172.21.15.40:0/3354375696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c0ffae0 0x7fce8c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 --2- 172.21.15.40:0/3354375696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c100cd0 0x7fce8c101120 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 -- 172.21.15.40:0/3354375696 >> 172.21.15.40:0/3354375696 conn(0x7fce8c0fb0e0 msgr2=0x7fce8c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:52.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 -- 172.21.15.40:0/3354375696 shutdown_connections 2023-07-14T11:11:52.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.692+0000 7fce915ff700 1 -- 172.21.15.40:0/3354375696 wait complete. 2023-07-14T11:11:52.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 Processor -- start 2023-07-14T11:11:52.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 -- start start 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c100cd0 0x7fce8c195de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 0x7fce8c19ce50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce8c19d420 con 0x7fce8c0ffae0 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce8c19d560 con 0x7fce8c101660 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce915ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce8c19d6f0 con 0x7fce8c100cd0 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce8affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce8affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35564/0 (socket says 172.21.15.40:35564) 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.693+0000 7fce8affd700 1 -- 172.21.15.40:0/1737172136 learned_addr learned my addr 172.21.15.40:0/1737172136 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8a7fc700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c100cd0 0x7fce8c195de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8b7fe700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 0x7fce8c19ce50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 -- 172.21.15.40:0/1737172136 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c100cd0 msgr2=0x7fce8c195de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c100cd0 0x7fce8c195de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 -- 172.21.15.40:0/1737172136 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 msgr2=0x7fce8c19ce50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 0x7fce8c19ce50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 -- 172.21.15.40:0/1737172136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce7c009520 con 0x7fce8c0ffae0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce8affd700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fce7c00ea50 tx=0x7fce7c00ed60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce7c00eee0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fce84009c40 con 0x7fce8c0ffae0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fce7c00cbd0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fce7c018680 con 0x7fce8c0ffae0 2023-07-14T11:11:52.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.694+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fce800095f0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.696+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fce8c1070c0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.697+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fce7c010430 con 0x7fce8c0ffae0 2023-07-14T11:11:52.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.699+0000 7fce73fff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 0x7fce740a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:52.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.699+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fce7c014070 con 0x7fce8c0ffae0 2023-07-14T11:11:52.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.699+0000 7fce8a7fc700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 0x7fce740a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:52.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.700+0000 7fce8a7fc700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 0x7fce740a3600 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fce8000b2e0 tx=0x7fce80005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:52.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.702+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fce7c0965a0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/52085767' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:52.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:52 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3891186161' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:52.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.914+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7fce8c061120 con 0x7fce8c0ffae0 2023-07-14T11:11:52.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.915+0000 7fce73fff700 1 -- 172.21.15.40:0/1737172136 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7fce7c099bc0 con 0x7fce8c0ffae0 2023-07-14T11:11:52.917 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:52.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 msgr2=0x7fce740a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 0x7fce740a3600 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fce8000b2e0 tx=0x7fce80005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 msgr2=0x7fce8c1958a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 crypto rx=0x7fce7c00ea50 tx=0x7fce7c00ed60 comp rx=0 tx=0).stop 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 shutdown_connections 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fce740a1150 0x7fce740a3600 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fce8c101660 0x7fce8c19ce50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce8c0ffae0 0x7fce8c1958a0 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 --2- 172.21.15.40:0/1737172136 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fce8c100cd0 0x7fce8c195de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:52.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 >> 172.21.15.40:0/1737172136 conn(0x7fce8c0fb0e0 msgr2=0x7fce8c104200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:52.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 shutdown_connections 2023-07-14T11:11:52.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:52.917+0000 7fce915ff700 1 -- 172.21.15.40:0/1737172136 wait complete. 2023-07-14T11:11:52.937 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:53.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/52085767' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:52 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3891186161' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:53.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/52085767' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:52 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3891186161' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 -- 172.21.15.40:0/1148360677 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc0ffaf0 msgr2=0x7fc6cc0fff00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/1148360677 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc0fff00 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fc6c80098f0 tx=0x7fc6c8009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 -- 172.21.15.40:0/1148360677 shutdown_connections 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/1148360677 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc101670 0x7fc6cc107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/1148360677 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc0fff00 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/1148360677 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc101130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 -- 172.21.15.40:0/1148360677 >> 172.21.15.40:0/1148360677 conn(0x7fc6cc0fb130 msgr2=0x7fc6cc0fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:53.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 -- 172.21.15.40:0/1148360677 shutdown_connections 2023-07-14T11:11:53.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.143+0000 7fc6d35ff700 1 -- 172.21.15.40:0/1148360677 wait complete. 2023-07-14T11:11:53.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 Processor -- start 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 -- start start 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 0x7fc6cc19f010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6cc19f5e0 con 0x7fc6cc100ce0 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6cc19f720 con 0x7fc6cc101670 2023-07-14T11:11:53.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.144+0000 7fc6d35ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6cc19f8b0 con 0x7fc6cc0ffaf0 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35590/0 (socket says 172.21.15.40:35590) 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 -- 172.21.15.40:0/799988996 learned_addr learned my addr 172.21.15.40:0/799988996 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d139b700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d1b9c700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 0x7fc6cc19f010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 -- 172.21.15.40:0/799988996 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc0ffaf0 msgr2=0x7fc6cc197a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc197a60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 -- 172.21.15.40:0/799988996 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 msgr2=0x7fc6cc19f010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 0x7fc6cc19f010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 -- 172.21.15.40:0/799988996 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6bc0095f0 con 0x7fc6cc100ce0 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d1b9c700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 0x7fc6cc19f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.145+0000 7fc6d0b9a700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc6bc00ec10 tx=0x7fc6bc00ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:53.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.146+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6bc00cd40 con 0x7fc6cc100ce0 2023-07-14T11:11:53.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.146+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6c8009520 con 0x7fc6cc100ce0 2023-07-14T11:11:53.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.146+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc6bc0044e0 con 0x7fc6cc100ce0 2023-07-14T11:11:53.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.146+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc6bc010690 con 0x7fc6cc100ce0 2023-07-14T11:11:53.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.146+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6cc19ff60 con 0x7fc6cc100ce0 2023-07-14T11:11:53.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.147+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc6cc192180 con 0x7fc6cc100ce0 2023-07-14T11:11:53.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.149+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc6bc003710 con 0x7fc6cc100ce0 2023-07-14T11:11:53.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.150+0000 7fc6c27fc700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 0x7fc6b80a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.151+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc6bc014070 con 0x7fc6cc100ce0 2023-07-14T11:11:53.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.151+0000 7fc6d139b700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 0x7fc6b80a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.151+0000 7fc6d139b700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 0x7fc6b80a3600 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fc6c80098c0 tx=0x7fc6c8019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:53.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.154+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc6bc0965e0 con 0x7fc6cc100ce0 2023-07-14T11:11:53.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.362+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7fc6cc061120 con 0x7fc6cc100ce0 2023-07-14T11:11:53.364 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.362+0000 7fc6c27fc700 1 -- 172.21.15.40:0/799988996 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7fc6bc099c00 con 0x7fc6cc100ce0 2023-07-14T11:11:53.364 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:53.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 msgr2=0x7fc6b80a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 0x7fc6b80a3600 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7fc6c80098c0 tx=0x7fc6c8019040 comp rx=0 tx=0).stop 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 msgr2=0x7fc6cc197fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 crypto rx=0x7fc6bc00ec10 tx=0x7fc6bc00ef20 comp rx=0 tx=0).stop 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 shutdown_connections 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc6b80a1150 0x7fc6b80a3600 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc6cc101670 0x7fc6cc19f010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc6cc0ffaf0 0x7fc6cc197a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 --2- 172.21.15.40:0/799988996 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc6cc100ce0 0x7fc6cc197fa0 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 >> 172.21.15.40:0/799988996 conn(0x7fc6cc0fb130 msgr2=0x7fc6cc106610 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:53.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 shutdown_connections 2023-07-14T11:11:53.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.365+0000 7fc6d35ff700 1 -- 172.21.15.40:0/799988996 wait complete. 2023-07-14T11:11:53.385 INFO:tasks.fwd_scrub:Wait for all MDSs to reach steady state... 2023-07-14T11:11:53.386 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:53.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 -- 172.21.15.40:0/1110847746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a8100cf0 msgr2=0x7f53a8101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/1110847746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a8100cf0 0x7f53a8101140 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 crypto rx=0x7f539c009990 tx=0x7f539c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 -- 172.21.15.40:0/1110847746 shutdown_connections 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/1110847746 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 0x7f53a8107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/1110847746 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a80ffb00 0x7f53a80fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/1110847746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a8100cf0 0x7f53a8101140 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.590+0000 7f53ad9ff700 1 -- 172.21.15.40:0/1110847746 >> 172.21.15.40:0/1110847746 conn(0x7f53a80fb0e0 msgr2=0x7f53a80fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.591+0000 7f53ad9ff700 1 -- 172.21.15.40:0/1110847746 shutdown_connections 2023-07-14T11:11:53.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.591+0000 7f53ad9ff700 1 -- 172.21.15.40:0/1110847746 wait complete. 2023-07-14T11:11:53.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 Processor -- start 2023-07-14T11:11:53.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 -- start start 2023-07-14T11:11:53.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a8100cf0 0x7f53a8197fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 0x7f53a819f030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53a819f600 con 0x7f53a80ffb00 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53a819f740 con 0x7f53a8101680 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53ad9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f53a819f8d0 con 0x7f53a8100cf0 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53a6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53a6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55528/0 (socket says 172.21.15.40:55528) 2023-07-14T11:11:53.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53a6ffd700 1 -- 172.21.15.40:0/3204742463 learned_addr learned my addr 172.21.15.40:0/3204742463 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53a77fe700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 0x7f53a819f030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.592+0000 7f53a67fc700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a8100cf0 0x7f53a8197fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 -- 172.21.15.40:0/3204742463 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a8100cf0 msgr2=0x7f53a8197fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a8100cf0 0x7f53a8197fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 -- 172.21.15.40:0/3204742463 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 msgr2=0x7f53a819f030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 0x7f53a819f030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 -- 172.21.15.40:0/3204742463 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f539c0095f0 con 0x7f53a80ffb00 2023-07-14T11:11:53.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53a6ffd700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f539800e970 tx=0x7f539800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:53.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5398009790 con 0x7f53a80ffb00 2023-07-14T11:11:53.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f539800c7e0 con 0x7f53a80ffb00 2023-07-14T11:11:53.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f53a819fbb0 con 0x7f53a80ffb00 2023-07-14T11:11:53.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.593+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5398003c70 con 0x7f53a80ffb00 2023-07-14T11:11:53.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.594+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53a81a00c0 con 0x7f53a80ffb00 2023-07-14T11:11:53.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.594+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f53a810b320 con 0x7f53a80ffb00 2023-07-14T11:11:53.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.597+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f539800cc50 con 0x7f53a80ffb00 2023-07-14T11:11:53.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.598+0000 7f53ac9fd700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 0x7f53900a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:53.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.598+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f5398014070 con 0x7f53a80ffb00 2023-07-14T11:11:53.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.598+0000 7f53a67fc700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 0x7f53900a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:53.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.599+0000 7f53a67fc700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 0x7f53900a3650 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f539c00b2e0 tx=0x7f539c005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:53.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.602+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f53980961f0 con 0x7f53a80ffb00 2023-07-14T11:11:53.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.860+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f53a81a0670 con 0x7f53a80ffb00 2023-07-14T11:11:53.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.861+0000 7f53ac9fd700 1 -- 172.21.15.40:0/3204742463 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f5398003dd0 con 0x7f53a80ffb00 2023-07-14T11:11:53.864 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:53.864 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 msgr2=0x7f53900a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 0x7f53900a3650 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f539c00b2e0 tx=0x7f539c005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 msgr2=0x7f53a8197a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f539800e970 tx=0x7f539800ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 shutdown_connections 2023-07-14T11:11:53.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f53900a11a0 0x7f53900a3650 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53a8101680 0x7f53a819f030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53a80ffb00 0x7f53a8197a80 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 --2- 172.21.15.40:0/3204742463 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53a8100cf0 0x7f53a8197fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 >> 172.21.15.40:0/3204742463 conn(0x7f53a80fb0e0 msgr2=0x7f53a8106560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 shutdown_connections 2023-07-14T11:11:53.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:53.864+0000 7f53ad9ff700 1 -- 172.21.15.40:0/3204742463 wait complete. 2023-07-14T11:11:53.869 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:53.885 INFO:tasks.fwd_scrub:Ready to start scrub thrashing 2023-07-14T11:11:53.885 INFO:tasks.fwd_scrub.ceph_manager:waiting for clean 2023-07-14T11:11:53.886 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-07-14T11:11:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:53 smithi130 ceph-mon[129405]: pgmap v345: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:53 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1737172136' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:53 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/799988996' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:53 smithi040 ceph-mon[97954]: pgmap v345: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:53 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1737172136' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:53 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/799988996' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:53 smithi045 ceph-mon[128791]: pgmap v345: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:53 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1737172136' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:53 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/799988996' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:54.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.086+0000 7fd0671ff700 1 -- 172.21.15.40:0/2116768798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060074550 msgr2=0x7fd0600729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.086+0000 7fd0671ff700 1 --2- 172.21.15.40:0/2116768798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060074550 0x7fd0600729a0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7fd054009990 tx=0x7fd054009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 -- 172.21.15.40:0/2116768798 shutdown_connections 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 --2- 172.21.15.40:0/2116768798 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 0x7fd060109e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 --2- 172.21.15.40:0/2116768798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060074550 0x7fd0600729a0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 --2- 172.21.15.40:0/2116768798 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060072f70 0x7fd0600733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 -- 172.21.15.40:0/2116768798 >> 172.21.15.40:0/2116768798 conn(0x7fd0600fb0e0 msgr2=0x7fd0600fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 -- 172.21.15.40:0/2116768798 shutdown_connections 2023-07-14T11:11:54.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.087+0000 7fd0671ff700 1 -- 172.21.15.40:0/2116768798 wait complete. 2023-07-14T11:11:54.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 Processor -- start 2023-07-14T11:11:54.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 -- start start 2023-07-14T11:11:54.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060074550 0x7fd060197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 0x7fd06019efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd06019f5a0 con 0x7fd060072f70 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd06019f6e0 con 0x7fd0601079c0 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd0671ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd06019f870 con 0x7fd060074550 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd064f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd064f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55548/0 (socket says 172.21.15.40:55548) 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd064f9b700 1 -- 172.21.15.40:0/196604862 learned_addr learned my addr 172.21.15.40:0/196604862 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:54.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.088+0000 7fd05ffff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060074550 0x7fd060197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd06579c700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 0x7fd06019efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 -- 172.21.15.40:0/196604862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060074550 msgr2=0x7fd060197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060074550 0x7fd060197f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 -- 172.21.15.40:0/196604862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 msgr2=0x7fd06019efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 0x7fd06019efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 -- 172.21.15.40:0/196604862 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0540095f0 con 0x7fd060072f70 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd064f9b700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fd054000c00 tx=0x7fd054005720 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:54.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd05401d070 con 0x7fd060072f70 2023-07-14T11:11:54.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd054022470 con 0x7fd060072f70 2023-07-14T11:11:54.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd06019faf0 con 0x7fd060072f70 2023-07-14T11:11:54.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.089+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd054017650 con 0x7fd060072f70 2023-07-14T11:11:54.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.090+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd06019ff10 con 0x7fd060072f70 2023-07-14T11:11:54.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.091+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd060106070 con 0x7fd060072f70 2023-07-14T11:11:54.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.093+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fd0540225e0 con 0x7fd060072f70 2023-07-14T11:11:54.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.094+0000 7fd05dffb700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 0x7fd0500a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.094+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fd0540cdf50 con 0x7fd060072f70 2023-07-14T11:11:54.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.094+0000 7fd05ffff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 0x7fd0500a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.095+0000 7fd05ffff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 0x7fd0500a3650 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fd060074270 tx=0x7fd04c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:54.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.098+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fd05405c790 con 0x7fd060072f70 2023-07-14T11:11:54.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.301+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fd060060950 con 0x7fd0500a11a0 2023-07-14T11:11:54.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.315+0000 7fd05dffb700 1 -- 172.21.15.40:0/196604862 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+426813 (secure 0 0 0) 0x7fd060060950 con 0x7fd0500a11a0 2023-07-14T11:11:54.322 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:54.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 msgr2=0x7fd0500a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 0x7fd0500a3650 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7fd060074270 tx=0x7fd04c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:54.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 msgr2=0x7fd060197a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7fd054000c00 tx=0x7fd054005720 comp rx=0 tx=0).stop 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 shutdown_connections 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fd0500a11a0 0x7fd0500a3650 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd0601079c0 0x7fd06019efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd060072f70 0x7fd060197a20 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 --2- 172.21.15.40:0/196604862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd060074550 0x7fd060197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 >> 172.21.15.40:0/196604862 conn(0x7fd0600fb0e0 msgr2=0x7fd0600fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:54.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 shutdown_connections 2023-07-14T11:11:54.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.320+0000 7fd0671ff700 1 -- 172.21.15.40:0/196604862 wait complete. 2023-07-14T11:11:54.326 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:11:54.341 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":345,"stamp":"2023-07-14T11:11:52.658518+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":600650,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":1115,"num_read_kb":625,"num_write":392,"num_write_kb":3007,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":322,"ondisk_log_size":322,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337620,"kb_used_data":14196,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604204,"statfs":{"total":1151940427776,"available":1151594704896,"internally_reserved":0,"allocated":14536704,"data_stored":7896756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":258,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":121,"num_read_kb":56,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002858"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289856+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:11:09.289856+0000","last_peered":"2023-07-14T11:11:09.289856+0000","last_clean":"2023-07-14T11:11:09.289856+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:11:09.289856+0000","last_undegraded":"2023-07-14T11:11:09.289856+0000","last_fullsized":"2023-07-14T11:11:09.289856+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:21:08.313051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289223+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:11:09.289223+0000","last_peered":"2023-07-14T11:11:09.289223+0000","last_clean":"2023-07-14T11:11:09.289223+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:11:09.289223+0000","last_undegraded":"2023-07-14T11:11:09.289223+0000","last_fullsized":"2023-07-14T11:11:09.289223+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:31.561921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289047+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:11:09.289047+0000","last_peered":"2023-07-14T11:11:09.289047+0000","last_clean":"2023-07-14T11:11:09.289047+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:11:09.289047+0000","last_undegraded":"2023-07-14T11:11:09.289047+0000","last_fullsized":"2023-07-14T11:11:09.289047+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:16:30.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418276+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:11:09.418276+0000","last_peered":"2023-07-14T11:11:09.418276+0000","last_clean":"2023-07-14T11:11:09.418276+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:11:09.418276+0000","last_undegraded":"2023-07-14T11:11:09.418276+0000","last_fullsized":"2023-07-14T11:11:09.418276+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:00:30.994129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722848+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:11:08.722848+0000","last_peered":"2023-07-14T11:11:08.722848+0000","last_clean":"2023-07-14T11:11:08.722848+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:11:08.722848+0000","last_undegraded":"2023-07-14T11:11:08.722848+0000","last_fullsized":"2023-07-14T11:11:08.722848+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:27.534998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722230+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:11:08.722230+0000","last_peered":"2023-07-14T11:11:08.722230+0000","last_clean":"2023-07-14T11:11:08.722230+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:11:08.722230+0000","last_undegraded":"2023-07-14T11:11:08.722230+0000","last_fullsized":"2023-07-14T11:11:08.722230+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:41.760376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289149+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:11:09.289149+0000","last_peered":"2023-07-14T11:11:09.289149+0000","last_clean":"2023-07-14T11:11:09.289149+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:11:09.289149+0000","last_undegraded":"2023-07-14T11:11:09.289149+0000","last_fullsized":"2023-07-14T11:11:09.289149+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:33.117233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719836+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:11:08.719836+0000","last_peered":"2023-07-14T11:11:08.719836+0000","last_clean":"2023-07-14T11:11:08.719836+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:11:08.719836+0000","last_undegraded":"2023-07-14T11:11:08.719836+0000","last_fullsized":"2023-07-14T11:11:08.719836+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:23:33.952593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001574+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:11:09.001574+0000","last_peered":"2023-07-14T11:11:09.001574+0000","last_clean":"2023-07-14T11:11:09.001574+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:11:09.001574+0000","last_undegraded":"2023-07-14T11:11:09.001574+0000","last_fullsized":"2023-07-14T11:11:09.001574+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:31.244279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721940+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:11:08.721940+0000","last_peered":"2023-07-14T11:11:08.721940+0000","last_clean":"2023-07-14T11:11:08.721940+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:11:08.721940+0000","last_undegraded":"2023-07-14T11:11:08.721940+0000","last_fullsized":"2023-07-14T11:11:08.721940+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:50:22.375884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289757+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:11:09.289757+0000","last_peered":"2023-07-14T11:11:09.289757+0000","last_clean":"2023-07-14T11:11:09.289757+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:11:09.289757+0000","last_undegraded":"2023-07-14T11:11:09.289757+0000","last_fullsized":"2023-07-14T11:11:09.289757+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:49:09.252229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418209+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:11:09.418209+0000","last_peered":"2023-07-14T11:11:09.418209+0000","last_clean":"2023-07-14T11:11:09.418209+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:11:09.418209+0000","last_undegraded":"2023-07-14T11:11:09.418209+0000","last_fullsized":"2023-07-14T11:11:09.418209+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:12:39.940446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719070+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:11:08.719070+0000","last_peered":"2023-07-14T11:11:08.719070+0000","last_clean":"2023-07-14T11:11:08.719070+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:11:08.719070+0000","last_undegraded":"2023-07-14T11:11:08.719070+0000","last_fullsized":"2023-07-14T11:11:08.719070+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:21:56.387394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288747+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:11:09.288747+0000","last_peered":"2023-07-14T11:11:09.288747+0000","last_clean":"2023-07-14T11:11:09.288747+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:11:09.288747+0000","last_undegraded":"2023-07-14T11:11:09.288747+0000","last_fullsized":"2023-07-14T11:11:09.288747+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:34.963471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288910+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:11:09.288910+0000","last_peered":"2023-07-14T11:11:09.288910+0000","last_clean":"2023-07-14T11:11:09.288910+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:11:09.288910+0000","last_undegraded":"2023-07-14T11:11:09.288910+0000","last_fullsized":"2023-07-14T11:11:09.288910+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:27:24.871920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000761+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:11:09.000761+0000","last_peered":"2023-07-14T11:11:09.000761+0000","last_clean":"2023-07-14T11:11:09.000761+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:11:09.000761+0000","last_undegraded":"2023-07-14T11:11:09.000761+0000","last_fullsized":"2023-07-14T11:11:09.000761+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:22:12.060207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289626+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:11:09.289626+0000","last_peered":"2023-07-14T11:11:09.289626+0000","last_clean":"2023-07-14T11:11:09.289626+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:11:09.289626+0000","last_undegraded":"2023-07-14T11:11:09.289626+0000","last_fullsized":"2023-07-14T11:11:09.289626+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:33:58.195208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290140+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:11:09.290140+0000","last_peered":"2023-07-14T11:11:09.290140+0000","last_clean":"2023-07-14T11:11:09.290140+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:11:09.290140+0000","last_undegraded":"2023-07-14T11:11:09.290140+0000","last_fullsized":"2023-07-14T11:11:09.290140+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:38.328569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290427+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:11:09.290427+0000","last_peered":"2023-07-14T11:11:09.290427+0000","last_clean":"2023-07-14T11:11:09.290427+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:09.290427+0000","last_undegraded":"2023-07-14T11:11:09.290427+0000","last_fullsized":"2023-07-14T11:11:09.290427+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:45:52.717942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290086+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:11:09.290086+0000","last_peered":"2023-07-14T11:11:09.290086+0000","last_clean":"2023-07-14T11:11:09.290086+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:11:09.290086+0000","last_undegraded":"2023-07-14T11:11:09.290086+0000","last_fullsized":"2023-07-14T11:11:09.290086+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:54:00.281074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"90'13","reported_seq":140,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.553095+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:11:50.553095+0000","last_peered":"2023-07-14T11:11:50.553095+0000","last_clean":"2023-07-14T11:11:50.553095+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:11:50.553095+0000","last_undegraded":"2023-07-14T11:11:50.553095+0000","last_fullsized":"2023-07-14T11:11:50.553095+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:46:31.832968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":96,"num_read_kb":96,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720370+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:11:08.720370+0000","last_peered":"2023-07-14T11:11:08.720370+0000","last_clean":"2023-07-14T11:11:08.720370+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:11:08.720370+0000","last_undegraded":"2023-07-14T11:11:08.720370+0000","last_fullsized":"2023-07-14T11:11:08.720370+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:41:19.828306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417507+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:11:09.417507+0000","last_peered":"2023-07-14T11:11:09.417507+0000","last_clean":"2023-07-14T11:11:09.417507+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:11:09.417507+0000","last_undegraded":"2023-07-14T11:11:09.417507+0000","last_fullsized":"2023-07-14T11:11:09.417507+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:22:30.970594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720636+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:11:08.720636+0000","last_peered":"2023-07-14T11:11:08.720636+0000","last_clean":"2023-07-14T11:11:08.720636+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:11:08.720636+0000","last_undegraded":"2023-07-14T11:11:08.720636+0000","last_fullsized":"2023-07-14T11:11:08.720636+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:28:23.282093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289308+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:11:09.289308+0000","last_peered":"2023-07-14T11:11:09.289308+0000","last_clean":"2023-07-14T11:11:09.289308+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:11:09.289308+0000","last_undegraded":"2023-07-14T11:11:09.289308+0000","last_fullsized":"2023-07-14T11:11:09.289308+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:14:50.981268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417664+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:11:09.417664+0000","last_peered":"2023-07-14T11:11:09.417664+0000","last_clean":"2023-07-14T11:11:09.417664+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:11:09.417664+0000","last_undegraded":"2023-07-14T11:11:09.417664+0000","last_fullsized":"2023-07-14T11:11:09.417664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:37.313270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'12","reported_seq":43,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723372+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:11:08.723372+0000","last_peered":"2023-07-14T11:11:08.723372+0000","last_clean":"2023-07-14T11:11:08.723372+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:11:08.723372+0000","last_undegraded":"2023-07-14T11:11:08.723372+0000","last_fullsized":"2023-07-14T11:11:08.723372+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:32.724455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001165+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:11:09.001165+0000","last_peered":"2023-07-14T11:11:09.001165+0000","last_clean":"2023-07-14T11:11:09.001165+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:11:09.001165+0000","last_undegraded":"2023-07-14T11:11:09.001165+0000","last_fullsized":"2023-07-14T11:11:09.001165+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:32:43.730998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417650+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:11:09.417650+0000","last_peered":"2023-07-14T11:11:09.417650+0000","last_clean":"2023-07-14T11:11:09.417650+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:11:09.417650+0000","last_undegraded":"2023-07-14T11:11:09.417650+0000","last_fullsized":"2023-07-14T11:11:09.417650+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:46.106349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723406+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:11:08.723406+0000","last_peered":"2023-07-14T11:11:08.723406+0000","last_clean":"2023-07-14T11:11:08.723406+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:11:08.723406+0000","last_undegraded":"2023-07-14T11:11:08.723406+0000","last_fullsized":"2023-07-14T11:11:08.723406+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:16:53.790679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719976+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:11:08.719976+0000","last_peered":"2023-07-14T11:11:08.719976+0000","last_clean":"2023-07-14T11:11:08.719976+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:11:08.719976+0000","last_undegraded":"2023-07-14T11:11:08.719976+0000","last_fullsized":"2023-07-14T11:11:08.719976+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:57:18.583351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290083+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:11:09.290083+0000","last_peered":"2023-07-14T11:11:09.290083+0000","last_clean":"2023-07-14T11:11:09.290083+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:11:09.290083+0000","last_undegraded":"2023-07-14T11:11:09.290083+0000","last_fullsized":"2023-07-14T11:11:09.290083+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:39:25.095450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"90'5","reported_seq":135,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.509424+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:11:50.509424+0000","last_peered":"2023-07-14T11:11:50.509424+0000","last_clean":"2023-07-14T11:11:50.509424+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:11:50.509424+0000","last_undegraded":"2023-07-14T11:11:50.509424+0000","last_fullsized":"2023-07-14T11:11:50.509424+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:51.482337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":100,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723544+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:11:08.723544+0000","last_peered":"2023-07-14T11:11:08.723544+0000","last_clean":"2023-07-14T11:11:08.723544+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:11:08.723544+0000","last_undegraded":"2023-07-14T11:11:08.723544+0000","last_fullsized":"2023-07-14T11:11:08.723544+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:35.444809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290389+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:11:09.290389+0000","last_peered":"2023-07-14T11:11:09.290389+0000","last_clean":"2023-07-14T11:11:09.290389+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:11:09.290389+0000","last_undegraded":"2023-07-14T11:11:09.290389+0000","last_fullsized":"2023-07-14T11:11:09.290389+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:40.201892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'7","reported_seq":37,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722127+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:11:08.722127+0000","last_peered":"2023-07-14T11:11:08.722127+0000","last_clean":"2023-07-14T11:11:08.722127+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:11:08.722127+0000","last_undegraded":"2023-07-14T11:11:08.722127+0000","last_fullsized":"2023-07-14T11:11:08.722127+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:59:36.711766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719793+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:11:08.719793+0000","last_peered":"2023-07-14T11:11:08.719793+0000","last_clean":"2023-07-14T11:11:08.719793+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:11:08.719793+0000","last_undegraded":"2023-07-14T11:11:08.719793+0000","last_fullsized":"2023-07-14T11:11:08.719793+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:20.276490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289569+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:11:09.289569+0000","last_peered":"2023-07-14T11:11:09.289569+0000","last_clean":"2023-07-14T11:11:09.289569+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:11:09.289569+0000","last_undegraded":"2023-07-14T11:11:09.289569+0000","last_fullsized":"2023-07-14T11:11:09.289569+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:52:51.277511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"90'3","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718943+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:11:08.718943+0000","last_peered":"2023-07-14T11:11:08.718943+0000","last_clean":"2023-07-14T11:11:08.718943+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:11:08.718943+0000","last_undegraded":"2023-07-14T11:11:08.718943+0000","last_fullsized":"2023-07-14T11:11:08.718943+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:11:40.222647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719691+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:11:08.719691+0000","last_peered":"2023-07-14T11:11:08.719691+0000","last_clean":"2023-07-14T11:11:08.719691+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:11:08.719691+0000","last_undegraded":"2023-07-14T11:11:08.719691+0000","last_fullsized":"2023-07-14T11:11:08.719691+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:36:10.169827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289202+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:11:09.289202+0000","last_peered":"2023-07-14T11:11:09.289202+0000","last_clean":"2023-07-14T11:11:09.289202+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:11:09.289202+0000","last_undegraded":"2023-07-14T11:11:09.289202+0000","last_fullsized":"2023-07-14T11:11:09.289202+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:08:58.024785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719453+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:11:08.719453+0000","last_peered":"2023-07-14T11:11:08.719453+0000","last_clean":"2023-07-14T11:11:08.719453+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:11:08.719453+0000","last_undegraded":"2023-07-14T11:11:08.719453+0000","last_fullsized":"2023-07-14T11:11:08.719453+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.383791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289719+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:11:09.289719+0000","last_peered":"2023-07-14T11:11:09.289719+0000","last_clean":"2023-07-14T11:11:09.289719+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:11:09.289719+0000","last_undegraded":"2023-07-14T11:11:09.289719+0000","last_fullsized":"2023-07-14T11:11:09.289719+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:36:52.817530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719974+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:11:08.719974+0000","last_peered":"2023-07-14T11:11:08.719974+0000","last_clean":"2023-07-14T11:11:08.719974+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:11:08.719974+0000","last_undegraded":"2023-07-14T11:11:08.719974+0000","last_fullsized":"2023-07-14T11:11:08.719974+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:20:52.884869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'4","reported_seq":222,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:48.562093+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:11:48.562093+0000","last_peered":"2023-07-14T11:11:48.562093+0000","last_clean":"2023-07-14T11:11:48.562093+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:11:48.562093+0000","last_undegraded":"2023-07-14T11:11:48.562093+0000","last_fullsized":"2023-07-14T11:11:48.562093+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:44:32.109970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":188,"num_read_kb":188,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001506+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:11:09.001506+0000","last_peered":"2023-07-14T11:11:09.001506+0000","last_clean":"2023-07-14T11:11:09.001506+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:11:09.001506+0000","last_undegraded":"2023-07-14T11:11:09.001506+0000","last_fullsized":"2023-07-14T11:11:09.001506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:15.291405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289640+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:11:09.289640+0000","last_peered":"2023-07-14T11:11:09.289640+0000","last_clean":"2023-07-14T11:11:09.289640+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:11:09.289640+0000","last_undegraded":"2023-07-14T11:11:09.289640+0000","last_fullsized":"2023-07-14T11:11:09.289640+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:45.979895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'4","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289621+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:11:09.289621+0000","last_peered":"2023-07-14T11:11:09.289621+0000","last_clean":"2023-07-14T11:11:09.289621+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:11:09.289621+0000","last_undegraded":"2023-07-14T11:11:09.289621+0000","last_fullsized":"2023-07-14T11:11:09.289621+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:56.170940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289258+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:11:09.289258+0000","last_peered":"2023-07-14T11:11:09.289258+0000","last_clean":"2023-07-14T11:11:09.289258+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:11:09.289258+0000","last_undegraded":"2023-07-14T11:11:09.289258+0000","last_fullsized":"2023-07-14T11:11:09.289258+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:08.488544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417841+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:11:09.417841+0000","last_peered":"2023-07-14T11:11:09.417841+0000","last_clean":"2023-07-14T11:11:09.417841+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:11:09.417841+0000","last_undegraded":"2023-07-14T11:11:09.417841+0000","last_fullsized":"2023-07-14T11:11:09.417841+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:28:53.190914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719886+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:11:08.719886+0000","last_peered":"2023-07-14T11:11:08.719886+0000","last_clean":"2023-07-14T11:11:08.719886+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:11:08.719886+0000","last_undegraded":"2023-07-14T11:11:08.719886+0000","last_fullsized":"2023-07-14T11:11:08.719886+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:22.302853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001306+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:11:09.001306+0000","last_peered":"2023-07-14T11:11:09.001306+0000","last_clean":"2023-07-14T11:11:09.001306+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:11:09.001306+0000","last_undegraded":"2023-07-14T11:11:09.001306+0000","last_fullsized":"2023-07-14T11:11:09.001306+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:55:41.153671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.293052+0000","last_change":"2023-07-14T11:11:09.292716+0000","last_active":"2023-07-14T11:11:09.293052+0000","last_peered":"2023-07-14T11:11:09.293052+0000","last_clean":"2023-07-14T11:11:09.293052+0000","last_became_active":"2023-07-14T11:11:09.292262+0000","last_became_peered":"2023-07-14T11:11:09.292262+0000","last_unstale":"2023-07-14T11:11:09.293052+0000","last_undegraded":"2023-07-14T11:11:09.293052+0000","last_fullsized":"2023-07-14T11:11:09.293052+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:44:07.811874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,2],"acting":[10,8,11,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"92'16","reported_seq":239,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:48.540436+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:11:48.540436+0000","last_peered":"2023-07-14T11:11:48.540436+0000","last_clean":"2023-07-14T11:11:48.540436+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:11:48.540436+0000","last_undegraded":"2023-07-14T11:11:48.540436+0000","last_fullsized":"2023-07-14T11:11:48.540436+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:44:43.522939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4052,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":193,"num_read_kb":101,"num_write":18,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723095+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:11:08.723095+0000","last_peered":"2023-07-14T11:11:08.723095+0000","last_clean":"2023-07-14T11:11:08.723095+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:11:08.723095+0000","last_undegraded":"2023-07-14T11:11:08.723095+0000","last_fullsized":"2023-07-14T11:11:08.723095+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:51:23.455018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720210+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:11:08.720210+0000","last_peered":"2023-07-14T11:11:08.720210+0000","last_clean":"2023-07-14T11:11:08.720210+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:11:08.720210+0000","last_undegraded":"2023-07-14T11:11:08.720210+0000","last_fullsized":"2023-07-14T11:11:08.720210+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:42.833169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"90'9","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290243+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:11:09.290243+0000","last_peered":"2023-07-14T11:11:09.290243+0000","last_clean":"2023-07-14T11:11:09.290243+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:11:09.290243+0000","last_undegraded":"2023-07-14T11:11:09.290243+0000","last_fullsized":"2023-07-14T11:11:09.290243+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:34.169535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719060+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:11:08.719060+0000","last_peered":"2023-07-14T11:11:08.719060+0000","last_clean":"2023-07-14T11:11:08.719060+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:11:08.719060+0000","last_undegraded":"2023-07-14T11:11:08.719060+0000","last_fullsized":"2023-07-14T11:11:08.719060+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:24.691976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290657+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:11:09.290657+0000","last_peered":"2023-07-14T11:11:09.290657+0000","last_clean":"2023-07-14T11:11:09.290657+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:11:09.290657+0000","last_undegraded":"2023-07-14T11:11:09.290657+0000","last_fullsized":"2023-07-14T11:11:09.290657+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:53:37.211874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720108+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:11:08.720108+0000","last_peered":"2023-07-14T11:11:08.720108+0000","last_clean":"2023-07-14T11:11:08.720108+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:11:08.720108+0000","last_undegraded":"2023-07-14T11:11:08.720108+0000","last_fullsized":"2023-07-14T11:11:08.720108+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:55:08.685860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289465+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:11:09.289465+0000","last_peered":"2023-07-14T11:11:09.289465+0000","last_clean":"2023-07-14T11:11:09.289465+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:11:09.289465+0000","last_undegraded":"2023-07-14T11:11:09.289465+0000","last_fullsized":"2023-07-14T11:11:09.289465+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:07:02.366836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290356+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:11:09.290356+0000","last_peered":"2023-07-14T11:11:09.290356+0000","last_clean":"2023-07-14T11:11:09.290356+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:11:09.290356+0000","last_undegraded":"2023-07-14T11:11:09.290356+0000","last_fullsized":"2023-07-14T11:11:09.290356+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:04:23.808018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.521863+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:11:50.521863+0000","last_peered":"2023-07-14T11:11:50.521863+0000","last_clean":"2023-07-14T11:11:50.521863+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:11:50.521863+0000","last_undegraded":"2023-07-14T11:11:50.521863+0000","last_fullsized":"2023-07-14T11:11:50.521863+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:08.806584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289179+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:11:09.289179+0000","last_peered":"2023-07-14T11:11:09.289179+0000","last_clean":"2023-07-14T11:11:09.289179+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:11:09.289179+0000","last_undegraded":"2023-07-14T11:11:09.289179+0000","last_fullsized":"2023-07-14T11:11:09.289179+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:00.991168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719386+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:11:08.719386+0000","last_peered":"2023-07-14T11:11:08.719386+0000","last_clean":"2023-07-14T11:11:08.719386+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:11:08.719386+0000","last_undegraded":"2023-07-14T11:11:08.719386+0000","last_fullsized":"2023-07-14T11:11:08.719386+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:10:03.730236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722670+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:11:08.722670+0000","last_peered":"2023-07-14T11:11:08.722670+0000","last_clean":"2023-07-14T11:11:08.722670+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:11:08.722670+0000","last_undegraded":"2023-07-14T11:11:08.722670+0000","last_fullsized":"2023-07-14T11:11:08.722670+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:46:28.743826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719745+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:11:08.719745+0000","last_peered":"2023-07-14T11:11:08.719745+0000","last_clean":"2023-07-14T11:11:08.719745+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:11:08.719745+0000","last_undegraded":"2023-07-14T11:11:08.719745+0000","last_fullsized":"2023-07-14T11:11:08.719745+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:26:00.226723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719524+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:11:08.719524+0000","last_peered":"2023-07-14T11:11:08.719524+0000","last_clean":"2023-07-14T11:11:08.719524+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:11:08.719524+0000","last_undegraded":"2023-07-14T11:11:08.719524+0000","last_fullsized":"2023-07-14T11:11:08.719524+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:30.579895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723335+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:11:08.723335+0000","last_peered":"2023-07-14T11:11:08.723335+0000","last_clean":"2023-07-14T11:11:08.723335+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:11:08.723335+0000","last_undegraded":"2023-07-14T11:11:08.723335+0000","last_fullsized":"2023-07-14T11:11:08.723335+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:23.398441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291267+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:11:09.291267+0000","last_peered":"2023-07-14T11:11:09.291267+0000","last_clean":"2023-07-14T11:11:09.291267+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:11:09.291267+0000","last_undegraded":"2023-07-14T11:11:09.291267+0000","last_fullsized":"2023-07-14T11:11:09.291267+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:13:20.237238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291884+0000","last_change":"2023-07-14T11:11:09.291884+0000","last_active":"2023-07-14T11:11:09.291884+0000","last_peered":"2023-07-14T11:11:09.291884+0000","last_clean":"2023-07-14T11:11:09.291884+0000","last_became_active":"2023-07-14T11:11:09.291383+0000","last_became_peered":"2023-07-14T11:11:09.291383+0000","last_unstale":"2023-07-14T11:11:09.291884+0000","last_undegraded":"2023-07-14T11:11:09.291884+0000","last_fullsized":"2023-07-14T11:11:09.291884+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:58:27.329172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,1],"acting":[8,10,11,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"90'1","reported_seq":32,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290030+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:11:09.290030+0000","last_peered":"2023-07-14T11:11:09.290030+0000","last_clean":"2023-07-14T11:11:09.290030+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:11:09.290030+0000","last_undegraded":"2023-07-14T11:11:09.290030+0000","last_fullsized":"2023-07-14T11:11:09.290030+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:49:08.911423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721739+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:11:08.721739+0000","last_peered":"2023-07-14T11:11:08.721739+0000","last_clean":"2023-07-14T11:11:08.721739+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:08.721739+0000","last_undegraded":"2023-07-14T11:11:08.721739+0000","last_fullsized":"2023-07-14T11:11:08.721739+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:02.329622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719406+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:11:08.719406+0000","last_peered":"2023-07-14T11:11:08.719406+0000","last_clean":"2023-07-14T11:11:08.719406+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:11:08.719406+0000","last_undegraded":"2023-07-14T11:11:08.719406+0000","last_fullsized":"2023-07-14T11:11:08.719406+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:54:56.816899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'5","reported_seq":130,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:49.552173+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:11:49.552173+0000","last_peered":"2023-07-14T11:11:49.552173+0000","last_clean":"2023-07-14T11:11:49.552173+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:11:49.552173+0000","last_undegraded":"2023-07-14T11:11:49.552173+0000","last_fullsized":"2023-07-14T11:11:49.552173+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:42.890747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":94,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721512+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:11:08.721512+0000","last_peered":"2023-07-14T11:11:08.721512+0000","last_clean":"2023-07-14T11:11:08.721512+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:11:08.721512+0000","last_undegraded":"2023-07-14T11:11:08.721512+0000","last_fullsized":"2023-07-14T11:11:08.721512+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.643656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719363+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:11:08.719363+0000","last_peered":"2023-07-14T11:11:08.719363+0000","last_clean":"2023-07-14T11:11:08.719363+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:11:08.719363+0000","last_undegraded":"2023-07-14T11:11:08.719363+0000","last_fullsized":"2023-07-14T11:11:08.719363+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:59:01.913296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719845+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:11:08.719845+0000","last_peered":"2023-07-14T11:11:08.719845+0000","last_clean":"2023-07-14T11:11:08.719845+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:11:08.719845+0000","last_undegraded":"2023-07-14T11:11:08.719845+0000","last_fullsized":"2023-07-14T11:11:08.719845+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:21:32.430525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'5","reported_seq":129,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:48.518425+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:11:48.518425+0000","last_peered":"2023-07-14T11:11:48.518425+0000","last_clean":"2023-07-14T11:11:48.518425+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:11:48.518425+0000","last_undegraded":"2023-07-14T11:11:48.518425+0000","last_fullsized":"2023-07-14T11:11:48.518425+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:37:13.796130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":93,"num_read_kb":93,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289479+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:11:09.289479+0000","last_peered":"2023-07-14T11:11:09.289479+0000","last_clean":"2023-07-14T11:11:09.289479+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:11:09.289479+0000","last_undegraded":"2023-07-14T11:11:09.289479+0000","last_fullsized":"2023-07-14T11:11:09.289479+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:41:35.094180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719797+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:11:08.719797+0000","last_peered":"2023-07-14T11:11:08.719797+0000","last_clean":"2023-07-14T11:11:08.719797+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:11:08.719797+0000","last_undegraded":"2023-07-14T11:11:08.719797+0000","last_fullsized":"2023-07-14T11:11:08.719797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:19:31.917453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":40,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417870+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:11:09.417870+0000","last_peered":"2023-07-14T11:11:09.417870+0000","last_clean":"2023-07-14T11:11:09.417870+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:11:09.417870+0000","last_undegraded":"2023-07-14T11:11:09.417870+0000","last_fullsized":"2023-07-14T11:11:09.417870+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:36:12.795737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":359,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289040+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:11:09.289040+0000","last_peered":"2023-07-14T11:11:09.289040+0000","last_clean":"2023-07-14T11:11:09.289040+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:11:09.289040+0000","last_undegraded":"2023-07-14T11:11:09.289040+0000","last_fullsized":"2023-07-14T11:11:09.289040+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:28:25.142388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720162+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:11:08.720162+0000","last_peered":"2023-07-14T11:11:08.720162+0000","last_clean":"2023-07-14T11:11:08.720162+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:11:08.720162+0000","last_undegraded":"2023-07-14T11:11:08.720162+0000","last_fullsized":"2023-07-14T11:11:08.720162+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:39:21.892266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718829+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:11:08.718829+0000","last_peered":"2023-07-14T11:11:08.718829+0000","last_clean":"2023-07-14T11:11:08.718829+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:11:08.718829+0000","last_undegraded":"2023-07-14T11:11:08.718829+0000","last_fullsized":"2023-07-14T11:11:08.718829+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:43:18.820300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'4","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289160+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:11:09.289160+0000","last_peered":"2023-07-14T11:11:09.289160+0000","last_clean":"2023-07-14T11:11:09.289160+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:11:09.289160+0000","last_undegraded":"2023-07-14T11:11:09.289160+0000","last_fullsized":"2023-07-14T11:11:09.289160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:59.013859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289968+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:11:09.289968+0000","last_peered":"2023-07-14T11:11:09.289968+0000","last_clean":"2023-07-14T11:11:09.289968+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:11:09.289968+0000","last_undegraded":"2023-07-14T11:11:09.289968+0000","last_fullsized":"2023-07-14T11:11:09.289968+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:29:24.636531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723306+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:11:08.723306+0000","last_peered":"2023-07-14T11:11:08.723306+0000","last_clean":"2023-07-14T11:11:08.723306+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:11:08.723306+0000","last_undegraded":"2023-07-14T11:11:08.723306+0000","last_fullsized":"2023-07-14T11:11:08.723306+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:37.536988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719233+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:11:08.719233+0000","last_peered":"2023-07-14T11:11:08.719233+0000","last_clean":"2023-07-14T11:11:08.719233+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:11:08.719233+0000","last_undegraded":"2023-07-14T11:11:08.719233+0000","last_fullsized":"2023-07-14T11:11:08.719233+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:03.090653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719888+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:11:08.719888+0000","last_peered":"2023-07-14T11:11:08.719888+0000","last_clean":"2023-07-14T11:11:08.719888+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:11:08.719888+0000","last_undegraded":"2023-07-14T11:11:08.719888+0000","last_fullsized":"2023-07-14T11:11:08.719888+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:53.362733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"90'7","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290126+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:11:09.290126+0000","last_peered":"2023-07-14T11:11:09.290126+0000","last_clean":"2023-07-14T11:11:09.290126+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:11:09.290126+0000","last_undegraded":"2023-07-14T11:11:09.290126+0000","last_fullsized":"2023-07-14T11:11:09.290126+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:27:46.858212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720461+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:11:08.720461+0000","last_peered":"2023-07-14T11:11:08.720461+0000","last_clean":"2023-07-14T11:11:08.720461+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:11:08.720461+0000","last_undegraded":"2023-07-14T11:11:08.720461+0000","last_fullsized":"2023-07-14T11:11:08.720461+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:05.284617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290123+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:11:09.290123+0000","last_peered":"2023-07-14T11:11:09.290123+0000","last_clean":"2023-07-14T11:11:09.290123+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:11:09.290123+0000","last_undegraded":"2023-07-14T11:11:09.290123+0000","last_fullsized":"2023-07-14T11:11:09.290123+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:02:54.462119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418279+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:11:09.418279+0000","last_peered":"2023-07-14T11:11:09.418279+0000","last_clean":"2023-07-14T11:11:09.418279+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:11:09.418279+0000","last_undegraded":"2023-07-14T11:11:09.418279+0000","last_fullsized":"2023-07-14T11:11:09.418279+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:28:55.029604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289318+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:11:09.289318+0000","last_peered":"2023-07-14T11:11:09.289318+0000","last_clean":"2023-07-14T11:11:09.289318+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:11:09.289318+0000","last_undegraded":"2023-07-14T11:11:09.289318+0000","last_fullsized":"2023-07-14T11:11:09.289318+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:35:10.693230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418030+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:11:09.418030+0000","last_peered":"2023-07-14T11:11:09.418030+0000","last_clean":"2023-07-14T11:11:09.418030+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:11:09.418030+0000","last_undegraded":"2023-07-14T11:11:09.418030+0000","last_fullsized":"2023-07-14T11:11:09.418030+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:44:16.733738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000828+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:11:09.000828+0000","last_peered":"2023-07-14T11:11:09.000828+0000","last_clean":"2023-07-14T11:11:09.000828+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:11:09.000828+0000","last_undegraded":"2023-07-14T11:11:09.000828+0000","last_fullsized":"2023-07-14T11:11:09.000828+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:33:19.700027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722662+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:11:08.722662+0000","last_peered":"2023-07-14T11:11:08.722662+0000","last_clean":"2023-07-14T11:11:08.722662+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:11:08.722662+0000","last_undegraded":"2023-07-14T11:11:08.722662+0000","last_fullsized":"2023-07-14T11:11:08.722662+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:14:20.212946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290230+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:11:09.290230+0000","last_peered":"2023-07-14T11:11:09.290230+0000","last_clean":"2023-07-14T11:11:09.290230+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:11:09.290230+0000","last_undegraded":"2023-07-14T11:11:09.290230+0000","last_fullsized":"2023-07-14T11:11:09.290230+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:02:32.312663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"90'7","reported_seq":37,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720135+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:11:08.720135+0000","last_peered":"2023-07-14T11:11:08.720135+0000","last_clean":"2023-07-14T11:11:08.720135+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:11:08.720135+0000","last_undegraded":"2023-07-14T11:11:08.720135+0000","last_fullsized":"2023-07-14T11:11:08.720135+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:55.458801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289387+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:11:09.289387+0000","last_peered":"2023-07-14T11:11:09.289387+0000","last_clean":"2023-07-14T11:11:09.289387+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:11:09.289387+0000","last_undegraded":"2023-07-14T11:11:09.289387+0000","last_fullsized":"2023-07-14T11:11:09.289387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:44.877318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417886+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:11:09.417886+0000","last_peered":"2023-07-14T11:11:09.417886+0000","last_clean":"2023-07-14T11:11:09.417886+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:11:09.417886+0000","last_undegraded":"2023-07-14T11:11:09.417886+0000","last_fullsized":"2023-07-14T11:11:09.417886+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:22:03.929374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290180+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:11:09.290180+0000","last_peered":"2023-07-14T11:11:09.290180+0000","last_clean":"2023-07-14T11:11:09.290180+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:11:09.290180+0000","last_undegraded":"2023-07-14T11:11:09.290180+0000","last_fullsized":"2023-07-14T11:11:09.290180+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:09.530818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720181+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:11:08.720181+0000","last_peered":"2023-07-14T11:11:08.720181+0000","last_clean":"2023-07-14T11:11:08.720181+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:11:08.720181+0000","last_undegraded":"2023-07-14T11:11:08.720181+0000","last_fullsized":"2023-07-14T11:11:08.720181+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:28:36.511332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417990+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:11:09.417990+0000","last_peered":"2023-07-14T11:11:09.417990+0000","last_clean":"2023-07-14T11:11:09.417990+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:11:09.417990+0000","last_undegraded":"2023-07-14T11:11:09.417990+0000","last_fullsized":"2023-07-14T11:11:09.417990+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:34:29.312985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":35,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722793+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:11:08.722793+0000","last_peered":"2023-07-14T11:11:08.722793+0000","last_clean":"2023-07-14T11:11:08.722793+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:11:08.722793+0000","last_undegraded":"2023-07-14T11:11:08.722793+0000","last_fullsized":"2023-07-14T11:11:08.722793+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:34:50.906895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289355+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:11:09.289355+0000","last_peered":"2023-07-14T11:11:09.289355+0000","last_clean":"2023-07-14T11:11:09.289355+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:11:09.289355+0000","last_undegraded":"2023-07-14T11:11:09.289355+0000","last_fullsized":"2023-07-14T11:11:09.289355+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:27:25.782530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000947+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:11:09.000947+0000","last_peered":"2023-07-14T11:11:09.000947+0000","last_clean":"2023-07-14T11:11:09.000947+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:11:09.000947+0000","last_undegraded":"2023-07-14T11:11:09.000947+0000","last_fullsized":"2023-07-14T11:11:09.000947+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:31:40.967816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.566007+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:11:50.566007+0000","last_peered":"2023-07-14T11:11:50.566007+0000","last_clean":"2023-07-14T11:11:50.566007+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:11:50.566007+0000","last_undegraded":"2023-07-14T11:11:50.566007+0000","last_fullsized":"2023-07-14T11:11:50.566007+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:28.815042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1122,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":97,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289850+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:11:09.289850+0000","last_peered":"2023-07-14T11:11:09.289850+0000","last_clean":"2023-07-14T11:11:09.289850+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:11:09.289850+0000","last_undegraded":"2023-07-14T11:11:09.289850+0000","last_fullsized":"2023-07-14T11:11:09.289850+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:30:10.951768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288876+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:11:09.288876+0000","last_peered":"2023-07-14T11:11:09.288876+0000","last_clean":"2023-07-14T11:11:09.288876+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:11:09.288876+0000","last_undegraded":"2023-07-14T11:11:09.288876+0000","last_fullsized":"2023-07-14T11:11:09.288876+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:35.079631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720056+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:11:08.720056+0000","last_peered":"2023-07-14T11:11:08.720056+0000","last_clean":"2023-07-14T11:11:08.720056+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:11:08.720056+0000","last_undegraded":"2023-07-14T11:11:08.720056+0000","last_fullsized":"2023-07-14T11:11:08.720056+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:24:42.663850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289490+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:11:09.289490+0000","last_peered":"2023-07-14T11:11:09.289490+0000","last_clean":"2023-07-14T11:11:09.289490+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:11:09.289490+0000","last_undegraded":"2023-07-14T11:11:09.289490+0000","last_fullsized":"2023-07-14T11:11:09.289490+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:58.994547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719226+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:11:08.719226+0000","last_peered":"2023-07-14T11:11:08.719226+0000","last_clean":"2023-07-14T11:11:08.719226+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:11:08.719226+0000","last_undegraded":"2023-07-14T11:11:08.719226+0000","last_fullsized":"2023-07-14T11:11:08.719226+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:14.394847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"90'5","reported_seq":45,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719234+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:11:08.719234+0000","last_peered":"2023-07-14T11:11:08.719234+0000","last_clean":"2023-07-14T11:11:08.719234+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:11:08.719234+0000","last_undegraded":"2023-07-14T11:11:08.719234+0000","last_fullsized":"2023-07-14T11:11:08.719234+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:47:15.179813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289653+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:11:09.289653+0000","last_peered":"2023-07-14T11:11:09.289653+0000","last_clean":"2023-07-14T11:11:09.289653+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:11:09.289653+0000","last_undegraded":"2023-07-14T11:11:09.289653+0000","last_fullsized":"2023-07-14T11:11:09.289653+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:11:51.768155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289543+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:11:09.289543+0000","last_peered":"2023-07-14T11:11:09.289543+0000","last_clean":"2023-07-14T11:11:09.289543+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:11:09.289543+0000","last_undegraded":"2023-07-14T11:11:09.289543+0000","last_fullsized":"2023-07-14T11:11:09.289543+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:14.076462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290470+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:11:09.290470+0000","last_peered":"2023-07-14T11:11:09.290470+0000","last_clean":"2023-07-14T11:11:09.290470+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:11:09.290470+0000","last_undegraded":"2023-07-14T11:11:09.290470+0000","last_fullsized":"2023-07-14T11:11:09.290470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:22:58.418691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290343+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:11:09.290343+0000","last_peered":"2023-07-14T11:11:09.290343+0000","last_clean":"2023-07-14T11:11:09.290343+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:11:09.290343+0000","last_undegraded":"2023-07-14T11:11:09.290343+0000","last_fullsized":"2023-07-14T11:11:09.290343+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:07:35.545905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289064+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:11:09.289064+0000","last_peered":"2023-07-14T11:11:09.289064+0000","last_clean":"2023-07-14T11:11:09.289064+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:11:09.289064+0000","last_undegraded":"2023-07-14T11:11:09.289064+0000","last_fullsized":"2023-07-14T11:11:09.289064+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:45:00.837634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719240+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:11:08.719240+0000","last_peered":"2023-07-14T11:11:08.719240+0000","last_clean":"2023-07-14T11:11:08.719240+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:11:08.719240+0000","last_undegraded":"2023-07-14T11:11:08.719240+0000","last_fullsized":"2023-07-14T11:11:08.719240+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:36.474008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291336+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:11:09.291336+0000","last_peered":"2023-07-14T11:11:09.291336+0000","last_clean":"2023-07-14T11:11:09.291336+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:11:09.291336+0000","last_undegraded":"2023-07-14T11:11:09.291336+0000","last_fullsized":"2023-07-14T11:11:09.291336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:26.293702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290764+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:11:09.290764+0000","last_peered":"2023-07-14T11:11:09.290764+0000","last_clean":"2023-07-14T11:11:09.290764+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:11:09.290764+0000","last_undegraded":"2023-07-14T11:11:09.290764+0000","last_fullsized":"2023-07-14T11:11:09.290764+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:45.932302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":15,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290111+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:11:09.290111+0000","last_peered":"2023-07-14T11:11:09.290111+0000","last_clean":"2023-07-14T11:11:09.290111+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:11:09.290111+0000","last_undegraded":"2023-07-14T11:11:09.290111+0000","last_fullsized":"2023-07-14T11:11:09.290111+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:00.414644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719489+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:11:08.719489+0000","last_peered":"2023-07-14T11:11:08.719489+0000","last_clean":"2023-07-14T11:11:08.719489+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:11:08.719489+0000","last_undegraded":"2023-07-14T11:11:08.719489+0000","last_fullsized":"2023-07-14T11:11:08.719489+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:03.664035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720295+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:11:08.720295+0000","last_peered":"2023-07-14T11:11:08.720295+0000","last_clean":"2023-07-14T11:11:08.720295+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:11:08.720295+0000","last_undegraded":"2023-07-14T11:11:08.720295+0000","last_fullsized":"2023-07-14T11:11:08.720295+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:05:07.837607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001328+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:11:09.001328+0000","last_peered":"2023-07-14T11:11:09.001328+0000","last_clean":"2023-07-14T11:11:09.001328+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:11:09.001328+0000","last_undegraded":"2023-07-14T11:11:09.001328+0000","last_fullsized":"2023-07-14T11:11:09.001328+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:01.143943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719197+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:11:08.719197+0000","last_peered":"2023-07-14T11:11:08.719197+0000","last_clean":"2023-07-14T11:11:08.719197+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:11:08.719197+0000","last_undegraded":"2023-07-14T11:11:08.719197+0000","last_fullsized":"2023-07-14T11:11:08.719197+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:14:33.943135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290872+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:11:09.290872+0000","last_peered":"2023-07-14T11:11:09.290872+0000","last_clean":"2023-07-14T11:11:09.290872+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:11:09.290872+0000","last_undegraded":"2023-07-14T11:11:09.290872+0000","last_fullsized":"2023-07-14T11:11:09.290872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:51:23.080257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":10282,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":979,"num_read_kb":506,"num_write":211,"num_write_kb":69,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":30846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185,"ondisk_log_size":185,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416423,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":1020,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":1044480,"data_stored":509181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54800000000000004,"15min":0.55600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.439,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83099999999999996,"5min":1.202,"15min":1.202},"last":0.61899999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64400000000000002,"15min":0.64100000000000001},"min":{"1min":0.52900000000000003,"5min":0.443,"15min":0.443},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.63100000000000001,"15min":0.627},"min":{"1min":0.49099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73499999999999999,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.379},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.35399999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.74199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.67100000000000004,"15min":0.67500000000000004},"min":{"1min":0.496,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81299999999999994,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.68300000000000005,"15min":0.68100000000000005},"min":{"1min":0.56200000000000006,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85099999999999998,"5min":1.05,"15min":1.05},"last":0.70399999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79200000000000004,"5min":1.254,"15min":1.254},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.61099999999999999,"15min":0.61299999999999999},"min":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84599999999999997,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71199999999999997,"15min":0.71299999999999997},"min":{"1min":0.495,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73799999999999999,"15min":0.74099999999999999},"min":{"1min":0.53600000000000003,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.92900000000000005,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.72799999999999998,"15min":0.72599999999999998},"min":{"1min":0.59699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.91400000000000003,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.57499999999999996,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.89500000000000002,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.80400000000000005}]},{"osd":8,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.754},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.53700000000000003,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84099999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78600000000000003,"15min":0.78700000000000003},"min":{"1min":0.61799999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":0.95299999999999996,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.80400000000000005,"15min":0.80200000000000005},"min":{"1min":0.58499999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.76900000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.82499999999999996,"15min":0.82199999999999995},"min":{"1min":0.64800000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0149999999999999,"5min":1.411,"15min":1.411},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.66700000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.99299999999999999,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.82499999999999996}]}]},{"osd":10,"up_from":72,"seq":309237645353,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28596,"kb_used_data":1644,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716556,"statfs":{"total":95995035648,"available":95965753344,"internally_reserved":0,"allocated":1683456,"data_stored":1104145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.82799999999999996,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.77700000000000002,"15min":0.79500000000000004},"min":{"1min":0.54500000000000004,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90500000000000003,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.64600000000000002}]},{"osd":1,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.46000000000000002,"15min":0.45600000000000002},"min":{"1min":0.33100000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59099999999999997,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.502,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.85399999999999998,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.375}]},{"osd":2,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.65800000000000003,"15min":0.64400000000000002},"min":{"1min":0.47199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91400000000000003,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.66300000000000003,"15min":0.66700000000000004},"min":{"1min":0.44400000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80000000000000004,"5min":1.796,"15min":1.796},"last":0.60099999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.80700000000000005,"15min":0.88100000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69599999999999995,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.66500000000000004,"15min":0.67300000000000004},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85199999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.48299999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85499999999999998,"5min":1.839,"15min":1.839},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.626,"5min":0.57599999999999996,"15min":0.56200000000000006},"min":{"1min":0.39900000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81000000000000005,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.73999999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.73399999999999999,"15min":0.72799999999999998},"min":{"1min":0.60099999999999998,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.90900000000000003,"5min":1.165,"15min":1.165},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.746,"5min":0.76400000000000001,"15min":0.76900000000000002},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":0.90100000000000002,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.79200000000000004}]},{"osd":6,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.77100000000000002,"15min":0.77900000000000003},"min":{"1min":0.621,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.88100000000000001,"5min":1.897,"15min":1.897},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.747,"5min":0.876,"15min":0.93000000000000005},"min":{"1min":0.55500000000000005,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96499999999999997,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.66800000000000004}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.78600000000000003,"15min":0.78400000000000003},"min":{"1min":0.63700000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.02,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.627},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.83699999999999997,"15min":0.84999999999999998},"min":{"1min":0.58699999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.999,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.85899999999999999}]},{"osd":9,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.84199999999999997,"15min":0.84599999999999997},"min":{"1min":0.64100000000000001,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0169999999999999,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.871,"15min":0.88700000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.98799999999999999,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.89300000000000002}]},{"osd":11,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.85299999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":0.98699999999999999,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.99199999999999999,"15min":1.0189999999999999},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.048,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":0.77300000000000002}]}]},{"osd":9,"up_from":67,"seq":287762808878,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.504,"15min":0.52600000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72299999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.59099999999999997,"15min":0.621},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.98199999999999998,"5min":1.488,"15min":1.488},"last":0.38500000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.75,"15min":0.76500000000000001},"min":{"1min":0.48799999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.302,"5min":1.754,"15min":1.754},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69399999999999995,"15min":0.70599999999999996},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.55500000000000005}]},{"osd":2,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.72899999999999998,"15min":0.75600000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.103,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.73799999999999999,"15min":0.73999999999999999},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.3109999999999999,"5min":5.3109999999999999,"15min":5.3109999999999999},"last":0.62},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.68000000000000005,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.751,"15min":0.755},"min":{"1min":0.59099999999999997,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.214,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.81100000000000005,"15min":0.81699999999999995},"min":{"1min":0.64500000000000002,"5min":0.498,"15min":0.498},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83099999999999996,"15min":0.86299999999999999},"min":{"1min":0.55100000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.98199999999999998,"5min":1.242,"15min":1.242},"last":0.497},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.80000000000000004,"15min":0.83499999999999996},"min":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.097,"5min":1.258,"15min":1.258},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":0.95799999999999996,"15min":0.95599999999999996},"min":{"1min":0.60799999999999998,"5min":0.495,"15min":0.495},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":5.2089999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92200000000000004,"15min":0.96499999999999997},"min":{"1min":0.621,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.5269999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.71099999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.87,"15min":0.90500000000000003},"min":{"1min":0.60199999999999998,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2609999999999999,"5min":1.331,"15min":1.331},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.95199999999999996,"15min":0.99199999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.196,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.53700000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.875,"15min":0.90400000000000003},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.298,"5min":1.7310000000000001,"15min":1.7310000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.137,"5min":1.002,"15min":0.98599999999999999},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":5.3579999999999997,"5min":5.3579999999999997,"15min":5.3579999999999997},"last":0.68200000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.82799999999999996,"15min":0.82499999999999996},"min":{"1min":0.59599999999999997,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.877,"15min":0.88200000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.64000000000000001}]}]},{"osd":8,"up_from":59,"seq":253403070515,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27992,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717160,"statfs":{"total":95995035648,"available":95966371840,"internally_reserved":0,"allocated":1064960,"data_stored":509537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83899999999999997,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59399999999999997,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.84899999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.55900000000000005,"15min":0.53600000000000003},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78200000000000003,"5min":0.875,"15min":0.875},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.61499999999999999,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79800000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.69599999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72299999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.71599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.40300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93000000000000005,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91800000000000004,"5min":1.079,"15min":1.079},"last":0.68700000000000006}]},{"osd":4,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.61899999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.89700000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.64800000000000002,"15min":0.63600000000000001},"min":{"1min":0.56799999999999995,"5min":0.37,"15min":0.37},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.73699999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.79800000000000004,"5min":1.052,"15min":1.052},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82699999999999996,"5min":1.135,"15min":1.135},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.70599999999999996,"15min":0.70799999999999996},"min":{"1min":0.48899999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90100000000000002,"5min":1.008,"15min":1.008},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.73599999999999999,"15min":0.74199999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.71699999999999997},"min":{"1min":0.58199999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":1.028,"15min":1.028},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.73799999999999999,"15min":0.73199999999999998},"min":{"1min":0.54500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.99399999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.629}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76400000000000001,"15min":0.75600000000000001},"min":{"1min":0.61599999999999999,"5min":0.433,"15min":0.433},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.79100000000000004,"15min":0.77900000000000003},"min":{"1min":0.63100000000000001,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0309999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.81599999999999995}]},{"osd":10,"last update":"Fri Jul 14 11:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.75700000000000001,"15min":0.72899999999999998},"min":{"1min":0.65800000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.753},{"interface":"front","average":{"1min":0.876,"5min":0.78700000000000003,"15min":0.753},"min":{"1min":0.65100000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.081,"5min":1.085,"15min":1.085},"last":0.85399999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.81999999999999995,"15min":0.81399999999999995},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.0569999999999999,"5min":1.117,"15min":1.117},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.79200000000000004,"15min":0.78400000000000003},"min":{"1min":0.67200000000000004,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.83699999999999997}]}]},{"osd":7,"up_from":51,"seq":219043332151,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1616,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1654784,"data_stored":1101130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.66300000000000003,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58599999999999997,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59299999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.64100000000000001,"15min":0.65200000000000002},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.92200000000000004,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":0.872,"15min":0.872},"last":0.625}]},{"osd":3,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.61599999999999999,"15min":0.58799999999999997},"min":{"1min":0.44400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.89200000000000002,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.63500000000000001,"15min":0.60299999999999998},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81899999999999995,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.78700000000000003,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69099999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.76600000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.70599999999999996,"15min":0.72199999999999998},"min":{"1min":0.55000000000000004,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.878,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.56899999999999995,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.88100000000000001,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.69099999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.74299999999999999,"15min":0.73799999999999999},"min":{"1min":0.47499999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.05,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.83199999999999996,"15min":0.75600000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":3.7810000000000001,"5min":7.3719999999999999,"15min":7.3719999999999999},"last":0.86199999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.76100000000000001,"15min":0.75900000000000001},"min":{"1min":0.65200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.96599999999999997,"5min":1,"15min":1},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72199999999999998,"15min":0.71999999999999997},"min":{"1min":0.625,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.746}]},{"osd":9,"last update":"Fri Jul 14 11:11:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.83599999999999997,"15min":0.81799999999999995},"min":{"1min":0.61199999999999999,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":3.7109999999999999,"5min":3.7109999999999999,"15min":3.7109999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.875,"15min":0.81899999999999995},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":7.2489999999999997,"5min":7.2489999999999997,"15min":7.2489999999999997},"last":0.89500000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.72999999999999998,"15min":0.71999999999999997},"min":{"1min":0.56799999999999995,"5min":0.442,"15min":0.442},"max":{"1min":0.91500000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.78900000000000003,"15min":0.79000000000000004},"min":{"1min":0.59999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.0580000000000001,"5min":1.216,"15min":1.216},"last":0.84599999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.83999999999999997,"15min":0.83599999999999997},"min":{"1min":0.70399999999999996,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":0.98599999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.877},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.85599999999999998,"15min":0.84999999999999998},"min":{"1min":0.79100000000000004,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.0129999999999999,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.92000000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495676,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.51000000000000001,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.64000000000000001,"15min":0.61399999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":0.59799999999999998}]},{"osd":1,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.0600000000000001,"5min":1.407,"15min":1.407},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.70399999999999996}]},{"osd":2,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.73699999999999999},"min":{"1min":0.60099999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.75900000000000001,"15min":0.72099999999999997},"min":{"1min":0.54700000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":4.8250000000000002},"last":0.625}]},{"osd":3,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.67100000000000004,"15min":0.64300000000000002},"min":{"1min":0.46700000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.623,"15min":0.60399999999999998},"min":{"1min":0.43099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.4540000000000002,"5min":2.4540000000000002,"15min":2.4540000000000002},"last":0.66900000000000004}]},{"osd":4,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76600000000000001,"15min":0.747},"min":{"1min":0.438,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.028,"5min":1.302,"15min":1.302},"last":0.90700000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.73399999999999999,"15min":0.74199999999999999},"min":{"1min":0.57499999999999996,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.84299999999999997,"15min":0.82399999999999995},"min":{"1min":0.61599999999999999,"5min":0.441,"15min":0.441},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.68799999999999994}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.84899999999999998,"15min":0.81000000000000005},"min":{"1min":0.51300000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.877},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.83099999999999996,"15min":0.81799999999999995},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.113,"5min":1.502,"15min":1.502},"last":0.86199999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.64800000000000002,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.93400000000000005,"15min":0.96599999999999997},"min":{"1min":0.62,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2270000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.82099999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.98499999999999999,"15min":1.0149999999999999},"min":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":2.577,"5min":2.577,"15min":2.577},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0009999999999999,"15min":1.0329999999999999},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":2.5630000000000002,"5min":2.5630000000000002,"15min":2.5630000000000002},"last":0.90000000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.96299999999999997,"15min":0.97899999999999998},"min":{"1min":0.71299999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.101,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.85199999999999998,"15min":0.86799999999999999},"min":{"1min":0.50800000000000001,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.998,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.81000000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:11:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.0349999999999999,"15min":1.012},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":4.6769999999999996,"5min":4.6769999999999996,"15min":4.6769999999999996},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.88300000000000001,"15min":0.88900000000000001},"min":{"1min":0.65300000000000002,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.101,"5min":1.248,"15min":1.248},"last":0.77700000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574933,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":513793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.432,"5min":0.253,"15min":0.253},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.60299999999999998,"15min":0.53100000000000003},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.66500000000000004}]},{"osd":2,"last update":"Fri Jul 14 11:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.66700000000000004,"15min":0.57099999999999995},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89200000000000002,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58999999999999997,"15min":0.52700000000000002},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.33900000000000002},"max":{"1min":0.82699999999999996,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.60299999999999998,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.863,"5min":1.863,"15min":1.863},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.71699999999999997,"15min":0.60899999999999999},"min":{"1min":0.45600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.8420000000000001,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.84699999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.71699999999999997,"15min":0.66100000000000003},"min":{"1min":0.56100000000000005,"5min":0.38,"15min":0.38},"max":{"1min":0.96499999999999997,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64400000000000002,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89600000000000002,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.69299999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.72799999999999998,"15min":0.67900000000000005},"min":{"1min":0.48699999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.4100000000000001,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.875},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.79500000000000004,"15min":0.73899999999999999},"min":{"1min":0.55600000000000005,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":2.0539999999999998,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":1.028}]},{"osd":7,"last update":"Fri Jul 14 11:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.71299999999999997,"15min":0.67400000000000004},"min":{"1min":0.42499999999999999,"5min":0.316,"15min":0.316},"max":{"1min":1.0880000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.65900000000000003,"15min":0.629},"min":{"1min":0.40500000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.94099999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.68999999999999995,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.01,"5min":1.04,"15min":1.04},"last":1.006},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.77800000000000002,"15min":0.72999999999999998},"min":{"1min":0.62,"5min":0.441,"15min":0.441},"max":{"1min":1.159,"5min":1.165,"15min":1.165},"last":0.97299999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.85199999999999998,"15min":0.73699999999999999},"min":{"1min":0.57699999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":6.6980000000000004,"5min":6.6980000000000004,"15min":6.6980000000000004},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.88100000000000001,"15min":0.77700000000000002},"min":{"1min":0.68400000000000005,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":6.593,"5min":6.593,"15min":6.593},"last":1.0549999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.83699999999999997,"15min":0.83199999999999996},"min":{"1min":0.45600000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.056,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.075},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73699999999999999,"15min":0.72099999999999997},"min":{"1min":0.47399999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.0720000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.73799999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.96499999999999997,"15min":0.97799999999999998},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.157,"5min":1.256,"15min":1.256},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.877,"15min":0.88800000000000001},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.83299999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738457,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.69699999999999995,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9039999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66000000000000003,"15min":0.57699999999999996},"min":{"1min":0.42999999999999999,"5min":0.36799999999999999,"15min":0.36499999999999999},"max":{"1min":1.875,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.44400000000000001}]},{"osd":2,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66200000000000003,"15min":0.58499999999999996},"min":{"1min":0.46400000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.93400000000000005,"5min":2.528,"15min":2.528},"last":0.875},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66900000000000004,"15min":0.60599999999999998},"min":{"1min":0.44800000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.88,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.73399999999999999}]},{"osd":3,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.71299999999999997,"15min":0.73299999999999998},"min":{"1min":0.437,"5min":0.433,"15min":0.433},"max":{"1min":0.82399999999999995,"5min":3.2599999999999998,"15min":3.2599999999999998},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.878,"15min":0.91500000000000004},"min":{"1min":0.45500000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.0920000000000001,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.84899999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.70599999999999996,"15min":0.63600000000000001},"min":{"1min":0.47199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.2829999999999999,"5min":2.48,"15min":2.48},"last":0.504},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.80000000000000004,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":5.9610000000000003,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":0.88800000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.81799999999999995,"15min":0.71499999999999997},"min":{"1min":0.55100000000000005,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.4019999999999999,"5min":6.0839999999999996,"15min":6.0839999999999996},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.75800000000000001,"15min":0.67000000000000004},"min":{"1min":0.62,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.02,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":0.96599999999999997}]},{"osd":6,"last update":"Fri Jul 14 11:11:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.77800000000000002,"15min":0.71999999999999997},"min":{"1min":0.433,"5min":0.38,"15min":0.38},"max":{"1min":6.1870000000000003,"5min":6.1870000000000003,"15min":6.1870000000000003},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.74299999999999999,"15min":0.71899999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.7450000000000001,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.68600000000000005}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.79500000000000004,"15min":0.79900000000000004},"min":{"1min":0.42599999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.0580000000000001,"5min":1.323,"15min":1.323},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.88100000000000001,"15min":0.90100000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91000000000000003,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.81899999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.91100000000000003,"15min":0.88200000000000001},"min":{"1min":0.55700000000000005,"5min":0.496,"15min":0.496},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.88100000000000001,"15min":0.89100000000000001},"min":{"1min":0.68600000000000005,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.93700000000000006}]},{"osd":9,"last update":"Fri Jul 14 11:11:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.82499999999999996,"15min":0.80500000000000005},"min":{"1min":0.50800000000000001,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.95399999999999996,"15min":0.91800000000000004},"min":{"1min":0.60299999999999998,"5min":0.437,"15min":0.437},"max":{"1min":6.1180000000000003,"5min":6.1180000000000003,"15min":6.1180000000000003},"last":0.98899999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.876,"15min":0.86699999999999999},"min":{"1min":0.57799999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.456,"5min":2.085,"15min":2.085},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.85799999999999998,"15min":0.82799999999999996},"min":{"1min":0.69799999999999995,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.137,"5min":1.359,"15min":1.359},"last":1.0029999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.95499999999999996,"15min":0.90000000000000002},"min":{"1min":0.77200000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":6.0460000000000003,"5min":6.0460000000000003,"15min":6.0460000000000003},"last":1.085},{"interface":"front","average":{"1min":1.026,"5min":0.88800000000000001,"15min":0.86499999999999999},"min":{"1min":0.752,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":1.02}]}]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28000,"kb_used_data":1048,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717152,"statfs":{"total":95995035648,"available":95966363648,"internally_reserved":0,"allocated":1073152,"data_stored":513692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.872,"5min":1.304,"15min":1.304},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.54200000000000004},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.33100000000000002},"max":{"1min":1.0229999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.44700000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.59099999999999997,"15min":0.54800000000000004},"min":{"1min":0.42499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.875,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.503},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.62,"15min":0.56599999999999995},"min":{"1min":0.54500000000000004,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.61899999999999999,"15min":0.60499999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.70599999999999996,"15min":0.64700000000000002},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":3.2210000000000001,"5min":3.2210000000000001,"15min":3.2210000000000001},"last":0.58199999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.72099999999999997,"15min":0.72599999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.74399999999999999,"15min":0.745},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.97999999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.624}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.46800000000000003},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73499999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.52600000000000002,"15min":0.48399999999999999},"min":{"1min":0.41399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78800000000000003,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Fri Jul 14 11:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.623,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.69199999999999995,"15min":0.64800000000000002},"min":{"1min":0.57199999999999995,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.83099999999999996}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.79400000000000004,"15min":0.79200000000000004},"min":{"1min":0.59699999999999998,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":0.995,"5min":0.998,"15min":0.998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.77200000000000002,"15min":0.77400000000000002},"min":{"1min":0.57699999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.94799999999999995,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.64900000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.60399999999999998,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.61499999999999999,"15min":0.58899999999999997},"min":{"1min":0.51800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.69499999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:11:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.76400000000000001,"15min":0.75},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.075,"5min":0.85699999999999998,"15min":0.81899999999999995},"min":{"1min":0.70299999999999996,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":3.27,"5min":3.27,"15min":3.27},"last":0.96299999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.80800000000000005,"15min":0.78500000000000003},"min":{"1min":0.64900000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0780000000000001,"5min":1.085,"15min":1.085},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.77100000000000002,"15min":0.74299999999999999},"min":{"1min":0.65600000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.879}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84299999999999997,"15min":0.84199999999999997},"min":{"1min":0.63300000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79000000000000004,"15min":0.78600000000000003},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.84999999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182473,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":508194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.48399999999999999,"15min":0.42099999999999999},"min":{"1min":0.379,"5min":0.252,"15min":0.252},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.44600000000000001},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.3120000000000001,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.41299999999999998}]},{"osd":1,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61599999999999999,"15min":0.59399999999999997},"min":{"1min":0.374,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.504},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.69299999999999995,"15min":0.60899999999999999},"min":{"1min":0.57799999999999996,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89400000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.626}]},{"osd":2,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.67300000000000004,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3879999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.749},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.68899999999999995,"15min":0.65100000000000002},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.88200000000000001}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.72799999999999998,"15min":0.65300000000000002},"min":{"1min":0.434,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.77200000000000002,"15min":0.69999999999999996},"min":{"1min":0.61399999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98199999999999998,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.45400000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.75900000000000001,"15min":0.73899999999999999},"min":{"1min":0.56499999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.92500000000000004,"5min":1.262,"15min":1.262},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.754,"15min":0.75},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1200000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.93000000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.748,"15min":0.72899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.044,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.755,"5min":0.75,"15min":0.75900000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.52100000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.81699999999999995,"15min":0.82699999999999996},"min":{"1min":0.53600000000000003,"5min":0.496,"15min":0.496},"max":{"1min":1.4670000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.83199999999999996,"15min":0.81599999999999995},"min":{"1min":0.53100000000000003,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.54700000000000004}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.89000000000000001,"15min":0.90800000000000003},"min":{"1min":0.46500000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.879,"5min":1.879,"15min":1.879},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.95299999999999996,"15min":0.98499999999999999},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.7490000000000001,"5min":1.7490000000000001,"15min":1.7490000000000001},"last":0.71399999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.98799999999999999,"15min":1.02},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.23,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.94299999999999995,"15min":0.96599999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.0009999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.68000000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.85799999999999998,"15min":0.86499999999999999},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.296,"5min":1.351,"15min":1.351},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.874,"5min":0.88200000000000001,"15min":0.88400000000000001},"min":{"1min":0.72399999999999998,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.014,"5min":1.163,"15min":1.163},"last":0.82999999999999996}]}]},{"osd":4,"up_from":33,"seq":141733920839,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39100000000000001,"15min":0.36799999999999999},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.47699999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.39200000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.59099999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52100000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89400000000000002,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.64200000000000002,"15min":0.58899999999999997},"min":{"1min":0.47999999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.874,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63300000000000001,"15min":0.59099999999999997},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.841,"5min":1.841,"15min":1.841},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.622,"15min":0.59199999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.857,"5min":1.857,"15min":1.857},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.64300000000000002,"15min":0.59599999999999997},"min":{"1min":0.46999999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69499999999999995,"15min":0.64600000000000002},"min":{"1min":0.60099999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.95199999999999996,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.63400000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.65300000000000002,"15min":0.63500000000000001},"min":{"1min":0.436,"5min":0.432,"15min":0.432},"max":{"1min":0.871,"5min":1.105,"15min":1.105},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71499999999999997,"15min":0.68200000000000005},"min":{"1min":0.49199999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97599999999999998,"5min":1.137,"15min":1.137},"last":0.76600000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.65400000000000003,"15min":0.61799999999999999},"min":{"1min":0.45000000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.99299999999999999,"5min":1.087,"15min":1.087},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.628,"15min":0.59799999999999998},"min":{"1min":0.47599999999999998,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.82299999999999995,"5min":1.034,"15min":1.034},"last":0.83699999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.73199999999999998,"15min":0.72799999999999998},"min":{"1min":0.54900000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.81000000000000005,"15min":0.80100000000000005},"min":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0329999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.877}]},{"osd":9,"last update":"Fri Jul 14 11:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.81499999999999995,"15min":0.84099999999999997},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":0.95299999999999996,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.81200000000000006,"15min":0.82399999999999995},"min":{"1min":0.64300000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0589999999999999,"5min":1.123,"15min":1.123},"last":0.79600000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86499999999999999,"15min":0.88300000000000001},"min":{"1min":0.65200000000000002,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.075,"5min":1.212,"15min":1.212},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.752,"5min":0.79100000000000004,"15min":0.80100000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93100000000000005,"5min":1.073,"15min":1.073},"last":0.90100000000000002}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.72099999999999997,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86799999999999999,"15min":0.86599999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.97999999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757314,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1667072,"data_stored":1100904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.40500000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.746,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54200000000000004,"15min":0.55400000000000005},"min":{"1min":0.36699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.75700000000000001,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52800000000000002}]},{"osd":1,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59099999999999997,"15min":0.58499999999999996},"min":{"1min":0.42699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.878,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.749},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61799999999999999,"15min":0.59599999999999997},"min":{"1min":0.53000000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83799999999999997,"5min":1.466,"15min":1.466},"last":0.64700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.34300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68999999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.45200000000000001,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60499999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.60399999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.60099999999999998},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41799999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78600000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.69299999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.71299999999999997,"15min":0.65900000000000003},"min":{"1min":0.49199999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88300000000000001,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.69599999999999995,"15min":0.64600000000000002},"min":{"1min":0.51700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90900000000000003,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65300000000000002,"15min":0.66300000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82199999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68600000000000005,"15min":0.68999999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.77100000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.93100000000000005,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.745,"5min":0.73599999999999999,"15min":0.73999999999999999},"min":{"1min":0.54900000000000004,"5min":0.496,"15min":0.496},"max":{"1min":0.93200000000000005,"5min":1.528,"15min":1.528},"last":0.91900000000000004}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.878,"15min":0.878},"last":0.90500000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75600000000000001,"15min":0.74399999999999999},"min":{"1min":0.49299999999999999,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.77200000000000002,"15min":0.76000000000000001},"min":{"1min":0.59199999999999997,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.88400000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.75600000000000001,"15min":0.73599999999999999},"min":{"1min":0.61899999999999999,"5min":0.439,"15min":0.439},"max":{"1min":0.95299999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.91000000000000003,"15min":0.96199999999999997},"min":{"1min":0.59499999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.93100000000000005,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.85099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":2772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":2998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":5740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:11:54.351 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-07-14T11:11:54.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.584+0000 7fc2587ff700 1 -- 172.21.15.40:0/1631997454 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc250100cf0 msgr2=0x7fc250101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.584+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1631997454 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc250100cf0 0x7fc250101140 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7fc2400098f0 tx=0x7fc240009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 -- 172.21.15.40:0/1631997454 shutdown_connections 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1631997454 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250101680 0x7fc250107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1631997454 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc2500ffb00 0x7fc2500fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1631997454 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc250100cf0 0x7fc250101140 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 -- 172.21.15.40:0/1631997454 >> 172.21.15.40:0/1631997454 conn(0x7fc2500fb0e0 msgr2=0x7fc2500fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:54.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 -- 172.21.15.40:0/1631997454 shutdown_connections 2023-07-14T11:11:54.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.585+0000 7fc2587ff700 1 -- 172.21.15.40:0/1631997454 wait complete. 2023-07-14T11:11:54.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 Processor -- start 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 -- start start 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 0x7fc250198040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc250101680 0x7fc25019f0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc25019f680 con 0x7fc2500ffb00 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc25019f7c0 con 0x7fc250100cf0 2023-07-14T11:11:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.586+0000 7fc2587ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc25019f950 con 0x7fc250101680 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55570/0 (socket says 172.21.15.40:55570) 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 -- 172.21.15.40:0/1111348527 learned_addr learned my addr 172.21.15.40:0/1111348527 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc255d9a700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 0x7fc250198040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc256d9c700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc250101680 0x7fc25019f0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 -- 172.21.15.40:0/1111348527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc250101680 msgr2=0x7fc25019f0b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc250101680 0x7fc25019f0b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 -- 172.21.15.40:0/1111348527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 msgr2=0x7fc250198040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 0x7fc250198040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 -- 172.21.15.40:0/1111348527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc248009d00 con 0x7fc2500ffb00 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc255d9a700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 0x7fc250198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc25659b700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc24c00e970 tx=0x7fc24c00ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.587+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc24c009790 con 0x7fc2500ffb00 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.588+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc240009520 con 0x7fc2500ffb00 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.588+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc24c00c7e0 con 0x7fc2500ffb00 2023-07-14T11:11:54.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.588+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc24c003c70 con 0x7fc2500ffb00 2023-07-14T11:11:54.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.588+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc2501a0000 con 0x7fc2500ffb00 2023-07-14T11:11:54.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.589+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc2501921e0 con 0x7fc2500ffb00 2023-07-14T11:11:54.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.591+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc24c00cc50 con 0x7fc2500ffb00 2023-07-14T11:11:54.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.592+0000 7fc2477fe700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 0x7fc23c0a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:54.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.592+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc24c014070 con 0x7fc2500ffb00 2023-07-14T11:11:54.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.592+0000 7fc255d9a700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 0x7fc23c0a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:54.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.593+0000 7fc255d9a700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 0x7fc23c0a36d0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fc240005e90 tx=0x7fc240019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:54.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.596+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc24c0961a0 con 0x7fc2500ffb00 2023-07-14T11:11:54.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.805+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fc250060950 con 0x7fc23c0a1220 2023-07-14T11:11:54.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.818+0000 7fc2477fe700 1 -- 172.21.15.40:0/1111348527 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+426843 (secure 0 0 0) 0x7fc250060950 con 0x7fc23c0a1220 2023-07-14T11:11:54.825 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:54.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 msgr2=0x7fc23c0a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 0x7fc23c0a36d0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fc240005e90 tx=0x7fc240019040 comp rx=0 tx=0).stop 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 msgr2=0x7fc250197b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 crypto rx=0x7fc24c00e970 tx=0x7fc24c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 shutdown_connections 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc23c0a1220 0x7fc23c0a36d0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc250101680 0x7fc25019f0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2500ffb00 0x7fc250197b00 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 --2- 172.21.15.40:0/1111348527 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc250100cf0 0x7fc250198040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:54.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 >> 172.21.15.40:0/1111348527 conn(0x7fc2500fb0e0 msgr2=0x7fc2501065c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:54.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 shutdown_connections 2023-07-14T11:11:54.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:54.824+0000 7fc2587ff700 1 -- 172.21.15.40:0/1111348527 wait complete. 2023-07-14T11:11:54.830 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:11:54.845 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":346,"stamp":"2023-07-14T11:11:54.659068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":600650,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":1140,"num_read_kb":645,"num_write":392,"num_write_kb":3007,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":322,"ondisk_log_size":322,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337620,"kb_used_data":14196,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604204,"statfs":{"total":1151940427776,"available":1151594704896,"internally_reserved":0,"allocated":14536704,"data_stored":7896756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":258,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":126,"num_read_kb":71,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002938"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289856+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:11:09.289856+0000","last_peered":"2023-07-14T11:11:09.289856+0000","last_clean":"2023-07-14T11:11:09.289856+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:11:09.289856+0000","last_undegraded":"2023-07-14T11:11:09.289856+0000","last_fullsized":"2023-07-14T11:11:09.289856+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:21:08.313051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289223+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:11:09.289223+0000","last_peered":"2023-07-14T11:11:09.289223+0000","last_clean":"2023-07-14T11:11:09.289223+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:11:09.289223+0000","last_undegraded":"2023-07-14T11:11:09.289223+0000","last_fullsized":"2023-07-14T11:11:09.289223+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:31.561921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289047+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:11:09.289047+0000","last_peered":"2023-07-14T11:11:09.289047+0000","last_clean":"2023-07-14T11:11:09.289047+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:11:09.289047+0000","last_undegraded":"2023-07-14T11:11:09.289047+0000","last_fullsized":"2023-07-14T11:11:09.289047+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:16:30.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418276+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:11:09.418276+0000","last_peered":"2023-07-14T11:11:09.418276+0000","last_clean":"2023-07-14T11:11:09.418276+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:11:09.418276+0000","last_undegraded":"2023-07-14T11:11:09.418276+0000","last_fullsized":"2023-07-14T11:11:09.418276+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:00:30.994129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722848+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:11:08.722848+0000","last_peered":"2023-07-14T11:11:08.722848+0000","last_clean":"2023-07-14T11:11:08.722848+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:11:08.722848+0000","last_undegraded":"2023-07-14T11:11:08.722848+0000","last_fullsized":"2023-07-14T11:11:08.722848+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:27.534998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722230+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:11:08.722230+0000","last_peered":"2023-07-14T11:11:08.722230+0000","last_clean":"2023-07-14T11:11:08.722230+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:11:08.722230+0000","last_undegraded":"2023-07-14T11:11:08.722230+0000","last_fullsized":"2023-07-14T11:11:08.722230+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:41.760376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289149+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:11:09.289149+0000","last_peered":"2023-07-14T11:11:09.289149+0000","last_clean":"2023-07-14T11:11:09.289149+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:11:09.289149+0000","last_undegraded":"2023-07-14T11:11:09.289149+0000","last_fullsized":"2023-07-14T11:11:09.289149+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:33.117233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719836+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:11:08.719836+0000","last_peered":"2023-07-14T11:11:08.719836+0000","last_clean":"2023-07-14T11:11:08.719836+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:11:08.719836+0000","last_undegraded":"2023-07-14T11:11:08.719836+0000","last_fullsized":"2023-07-14T11:11:08.719836+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:23:33.952593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001574+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:11:09.001574+0000","last_peered":"2023-07-14T11:11:09.001574+0000","last_clean":"2023-07-14T11:11:09.001574+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:11:09.001574+0000","last_undegraded":"2023-07-14T11:11:09.001574+0000","last_fullsized":"2023-07-14T11:11:09.001574+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:31.244279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721940+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:11:08.721940+0000","last_peered":"2023-07-14T11:11:08.721940+0000","last_clean":"2023-07-14T11:11:08.721940+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:11:08.721940+0000","last_undegraded":"2023-07-14T11:11:08.721940+0000","last_fullsized":"2023-07-14T11:11:08.721940+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:50:22.375884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289757+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:11:09.289757+0000","last_peered":"2023-07-14T11:11:09.289757+0000","last_clean":"2023-07-14T11:11:09.289757+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:11:09.289757+0000","last_undegraded":"2023-07-14T11:11:09.289757+0000","last_fullsized":"2023-07-14T11:11:09.289757+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:49:09.252229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418209+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:11:09.418209+0000","last_peered":"2023-07-14T11:11:09.418209+0000","last_clean":"2023-07-14T11:11:09.418209+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:11:09.418209+0000","last_undegraded":"2023-07-14T11:11:09.418209+0000","last_fullsized":"2023-07-14T11:11:09.418209+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:12:39.940446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719070+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:11:08.719070+0000","last_peered":"2023-07-14T11:11:08.719070+0000","last_clean":"2023-07-14T11:11:08.719070+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:11:08.719070+0000","last_undegraded":"2023-07-14T11:11:08.719070+0000","last_fullsized":"2023-07-14T11:11:08.719070+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:21:56.387394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288747+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:11:09.288747+0000","last_peered":"2023-07-14T11:11:09.288747+0000","last_clean":"2023-07-14T11:11:09.288747+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:11:09.288747+0000","last_undegraded":"2023-07-14T11:11:09.288747+0000","last_fullsized":"2023-07-14T11:11:09.288747+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:34.963471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288910+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:11:09.288910+0000","last_peered":"2023-07-14T11:11:09.288910+0000","last_clean":"2023-07-14T11:11:09.288910+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:11:09.288910+0000","last_undegraded":"2023-07-14T11:11:09.288910+0000","last_fullsized":"2023-07-14T11:11:09.288910+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:27:24.871920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000761+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:11:09.000761+0000","last_peered":"2023-07-14T11:11:09.000761+0000","last_clean":"2023-07-14T11:11:09.000761+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:11:09.000761+0000","last_undegraded":"2023-07-14T11:11:09.000761+0000","last_fullsized":"2023-07-14T11:11:09.000761+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:22:12.060207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289626+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:11:09.289626+0000","last_peered":"2023-07-14T11:11:09.289626+0000","last_clean":"2023-07-14T11:11:09.289626+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:11:09.289626+0000","last_undegraded":"2023-07-14T11:11:09.289626+0000","last_fullsized":"2023-07-14T11:11:09.289626+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:33:58.195208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290140+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:11:09.290140+0000","last_peered":"2023-07-14T11:11:09.290140+0000","last_clean":"2023-07-14T11:11:09.290140+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:11:09.290140+0000","last_undegraded":"2023-07-14T11:11:09.290140+0000","last_fullsized":"2023-07-14T11:11:09.290140+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:38.328569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290427+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:11:09.290427+0000","last_peered":"2023-07-14T11:11:09.290427+0000","last_clean":"2023-07-14T11:11:09.290427+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:09.290427+0000","last_undegraded":"2023-07-14T11:11:09.290427+0000","last_fullsized":"2023-07-14T11:11:09.290427+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:45:52.717942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290086+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:11:09.290086+0000","last_peered":"2023-07-14T11:11:09.290086+0000","last_clean":"2023-07-14T11:11:09.290086+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:11:09.290086+0000","last_undegraded":"2023-07-14T11:11:09.290086+0000","last_fullsized":"2023-07-14T11:11:09.290086+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:54:00.281074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"90'13","reported_seq":140,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.553095+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:11:50.553095+0000","last_peered":"2023-07-14T11:11:50.553095+0000","last_clean":"2023-07-14T11:11:50.553095+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:11:50.553095+0000","last_undegraded":"2023-07-14T11:11:50.553095+0000","last_fullsized":"2023-07-14T11:11:50.553095+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:46:31.832968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":96,"num_read_kb":96,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720370+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:11:08.720370+0000","last_peered":"2023-07-14T11:11:08.720370+0000","last_clean":"2023-07-14T11:11:08.720370+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:11:08.720370+0000","last_undegraded":"2023-07-14T11:11:08.720370+0000","last_fullsized":"2023-07-14T11:11:08.720370+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:41:19.828306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417507+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:11:09.417507+0000","last_peered":"2023-07-14T11:11:09.417507+0000","last_clean":"2023-07-14T11:11:09.417507+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:11:09.417507+0000","last_undegraded":"2023-07-14T11:11:09.417507+0000","last_fullsized":"2023-07-14T11:11:09.417507+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:22:30.970594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720636+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:11:08.720636+0000","last_peered":"2023-07-14T11:11:08.720636+0000","last_clean":"2023-07-14T11:11:08.720636+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:11:08.720636+0000","last_undegraded":"2023-07-14T11:11:08.720636+0000","last_fullsized":"2023-07-14T11:11:08.720636+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:28:23.282093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289308+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:11:09.289308+0000","last_peered":"2023-07-14T11:11:09.289308+0000","last_clean":"2023-07-14T11:11:09.289308+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:11:09.289308+0000","last_undegraded":"2023-07-14T11:11:09.289308+0000","last_fullsized":"2023-07-14T11:11:09.289308+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:14:50.981268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417664+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:11:09.417664+0000","last_peered":"2023-07-14T11:11:09.417664+0000","last_clean":"2023-07-14T11:11:09.417664+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:11:09.417664+0000","last_undegraded":"2023-07-14T11:11:09.417664+0000","last_fullsized":"2023-07-14T11:11:09.417664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:37.313270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'12","reported_seq":43,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723372+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:11:08.723372+0000","last_peered":"2023-07-14T11:11:08.723372+0000","last_clean":"2023-07-14T11:11:08.723372+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:11:08.723372+0000","last_undegraded":"2023-07-14T11:11:08.723372+0000","last_fullsized":"2023-07-14T11:11:08.723372+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:32.724455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001165+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:11:09.001165+0000","last_peered":"2023-07-14T11:11:09.001165+0000","last_clean":"2023-07-14T11:11:09.001165+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:11:09.001165+0000","last_undegraded":"2023-07-14T11:11:09.001165+0000","last_fullsized":"2023-07-14T11:11:09.001165+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:32:43.730998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417650+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:11:09.417650+0000","last_peered":"2023-07-14T11:11:09.417650+0000","last_clean":"2023-07-14T11:11:09.417650+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:11:09.417650+0000","last_undegraded":"2023-07-14T11:11:09.417650+0000","last_fullsized":"2023-07-14T11:11:09.417650+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:46.106349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723406+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:11:08.723406+0000","last_peered":"2023-07-14T11:11:08.723406+0000","last_clean":"2023-07-14T11:11:08.723406+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:11:08.723406+0000","last_undegraded":"2023-07-14T11:11:08.723406+0000","last_fullsized":"2023-07-14T11:11:08.723406+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:16:53.790679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719976+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:11:08.719976+0000","last_peered":"2023-07-14T11:11:08.719976+0000","last_clean":"2023-07-14T11:11:08.719976+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:11:08.719976+0000","last_undegraded":"2023-07-14T11:11:08.719976+0000","last_fullsized":"2023-07-14T11:11:08.719976+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:57:18.583351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290083+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:11:09.290083+0000","last_peered":"2023-07-14T11:11:09.290083+0000","last_clean":"2023-07-14T11:11:09.290083+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:11:09.290083+0000","last_undegraded":"2023-07-14T11:11:09.290083+0000","last_fullsized":"2023-07-14T11:11:09.290083+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:39:25.095450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"90'5","reported_seq":135,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.509424+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:11:50.509424+0000","last_peered":"2023-07-14T11:11:50.509424+0000","last_clean":"2023-07-14T11:11:50.509424+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:11:50.509424+0000","last_undegraded":"2023-07-14T11:11:50.509424+0000","last_fullsized":"2023-07-14T11:11:50.509424+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:51.482337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":100,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723544+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:11:08.723544+0000","last_peered":"2023-07-14T11:11:08.723544+0000","last_clean":"2023-07-14T11:11:08.723544+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:11:08.723544+0000","last_undegraded":"2023-07-14T11:11:08.723544+0000","last_fullsized":"2023-07-14T11:11:08.723544+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:35.444809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290389+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:11:09.290389+0000","last_peered":"2023-07-14T11:11:09.290389+0000","last_clean":"2023-07-14T11:11:09.290389+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:11:09.290389+0000","last_undegraded":"2023-07-14T11:11:09.290389+0000","last_fullsized":"2023-07-14T11:11:09.290389+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:40.201892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'7","reported_seq":37,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722127+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:11:08.722127+0000","last_peered":"2023-07-14T11:11:08.722127+0000","last_clean":"2023-07-14T11:11:08.722127+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:11:08.722127+0000","last_undegraded":"2023-07-14T11:11:08.722127+0000","last_fullsized":"2023-07-14T11:11:08.722127+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:59:36.711766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719793+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:11:08.719793+0000","last_peered":"2023-07-14T11:11:08.719793+0000","last_clean":"2023-07-14T11:11:08.719793+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:11:08.719793+0000","last_undegraded":"2023-07-14T11:11:08.719793+0000","last_fullsized":"2023-07-14T11:11:08.719793+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:20.276490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289569+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:11:09.289569+0000","last_peered":"2023-07-14T11:11:09.289569+0000","last_clean":"2023-07-14T11:11:09.289569+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:11:09.289569+0000","last_undegraded":"2023-07-14T11:11:09.289569+0000","last_fullsized":"2023-07-14T11:11:09.289569+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:52:51.277511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"90'3","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718943+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:11:08.718943+0000","last_peered":"2023-07-14T11:11:08.718943+0000","last_clean":"2023-07-14T11:11:08.718943+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:11:08.718943+0000","last_undegraded":"2023-07-14T11:11:08.718943+0000","last_fullsized":"2023-07-14T11:11:08.718943+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:11:40.222647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719691+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:11:08.719691+0000","last_peered":"2023-07-14T11:11:08.719691+0000","last_clean":"2023-07-14T11:11:08.719691+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:11:08.719691+0000","last_undegraded":"2023-07-14T11:11:08.719691+0000","last_fullsized":"2023-07-14T11:11:08.719691+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:36:10.169827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289202+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:11:09.289202+0000","last_peered":"2023-07-14T11:11:09.289202+0000","last_clean":"2023-07-14T11:11:09.289202+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:11:09.289202+0000","last_undegraded":"2023-07-14T11:11:09.289202+0000","last_fullsized":"2023-07-14T11:11:09.289202+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:08:58.024785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719453+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:11:08.719453+0000","last_peered":"2023-07-14T11:11:08.719453+0000","last_clean":"2023-07-14T11:11:08.719453+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:11:08.719453+0000","last_undegraded":"2023-07-14T11:11:08.719453+0000","last_fullsized":"2023-07-14T11:11:08.719453+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.383791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289719+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:11:09.289719+0000","last_peered":"2023-07-14T11:11:09.289719+0000","last_clean":"2023-07-14T11:11:09.289719+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:11:09.289719+0000","last_undegraded":"2023-07-14T11:11:09.289719+0000","last_fullsized":"2023-07-14T11:11:09.289719+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:36:52.817530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719974+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:11:08.719974+0000","last_peered":"2023-07-14T11:11:08.719974+0000","last_clean":"2023-07-14T11:11:08.719974+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:11:08.719974+0000","last_undegraded":"2023-07-14T11:11:08.719974+0000","last_fullsized":"2023-07-14T11:11:08.719974+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:20:52.884869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'4","reported_seq":232,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:53.569582+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:11:53.569582+0000","last_peered":"2023-07-14T11:11:53.569582+0000","last_clean":"2023-07-14T11:11:53.569582+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:11:53.569582+0000","last_undegraded":"2023-07-14T11:11:53.569582+0000","last_fullsized":"2023-07-14T11:11:53.569582+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:44:32.109970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":198,"num_read_kb":198,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001506+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:11:09.001506+0000","last_peered":"2023-07-14T11:11:09.001506+0000","last_clean":"2023-07-14T11:11:09.001506+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:11:09.001506+0000","last_undegraded":"2023-07-14T11:11:09.001506+0000","last_fullsized":"2023-07-14T11:11:09.001506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:15.291405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289640+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:11:09.289640+0000","last_peered":"2023-07-14T11:11:09.289640+0000","last_clean":"2023-07-14T11:11:09.289640+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:11:09.289640+0000","last_undegraded":"2023-07-14T11:11:09.289640+0000","last_fullsized":"2023-07-14T11:11:09.289640+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:45.979895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'4","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289621+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:11:09.289621+0000","last_peered":"2023-07-14T11:11:09.289621+0000","last_clean":"2023-07-14T11:11:09.289621+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:11:09.289621+0000","last_undegraded":"2023-07-14T11:11:09.289621+0000","last_fullsized":"2023-07-14T11:11:09.289621+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:56.170940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289258+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:11:09.289258+0000","last_peered":"2023-07-14T11:11:09.289258+0000","last_clean":"2023-07-14T11:11:09.289258+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:11:09.289258+0000","last_undegraded":"2023-07-14T11:11:09.289258+0000","last_fullsized":"2023-07-14T11:11:09.289258+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:08.488544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417841+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:11:09.417841+0000","last_peered":"2023-07-14T11:11:09.417841+0000","last_clean":"2023-07-14T11:11:09.417841+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:11:09.417841+0000","last_undegraded":"2023-07-14T11:11:09.417841+0000","last_fullsized":"2023-07-14T11:11:09.417841+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:28:53.190914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719886+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:11:08.719886+0000","last_peered":"2023-07-14T11:11:08.719886+0000","last_clean":"2023-07-14T11:11:08.719886+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:11:08.719886+0000","last_undegraded":"2023-07-14T11:11:08.719886+0000","last_fullsized":"2023-07-14T11:11:08.719886+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:22.302853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001306+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:11:09.001306+0000","last_peered":"2023-07-14T11:11:09.001306+0000","last_clean":"2023-07-14T11:11:09.001306+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:11:09.001306+0000","last_undegraded":"2023-07-14T11:11:09.001306+0000","last_fullsized":"2023-07-14T11:11:09.001306+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:55:41.153671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.293052+0000","last_change":"2023-07-14T11:11:09.292716+0000","last_active":"2023-07-14T11:11:09.293052+0000","last_peered":"2023-07-14T11:11:09.293052+0000","last_clean":"2023-07-14T11:11:09.293052+0000","last_became_active":"2023-07-14T11:11:09.292262+0000","last_became_peered":"2023-07-14T11:11:09.292262+0000","last_unstale":"2023-07-14T11:11:09.293052+0000","last_undegraded":"2023-07-14T11:11:09.293052+0000","last_fullsized":"2023-07-14T11:11:09.293052+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:44:07.811874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,2],"acting":[10,8,11,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"92'16","reported_seq":249,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:53.548802+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:11:53.548802+0000","last_peered":"2023-07-14T11:11:53.548802+0000","last_clean":"2023-07-14T11:11:53.548802+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:11:53.548802+0000","last_undegraded":"2023-07-14T11:11:53.548802+0000","last_fullsized":"2023-07-14T11:11:53.548802+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:44:43.522939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4052,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":203,"num_read_kb":106,"num_write":18,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723095+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:11:08.723095+0000","last_peered":"2023-07-14T11:11:08.723095+0000","last_clean":"2023-07-14T11:11:08.723095+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:11:08.723095+0000","last_undegraded":"2023-07-14T11:11:08.723095+0000","last_fullsized":"2023-07-14T11:11:08.723095+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:51:23.455018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720210+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:11:08.720210+0000","last_peered":"2023-07-14T11:11:08.720210+0000","last_clean":"2023-07-14T11:11:08.720210+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:11:08.720210+0000","last_undegraded":"2023-07-14T11:11:08.720210+0000","last_fullsized":"2023-07-14T11:11:08.720210+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:42.833169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"90'9","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290243+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:11:09.290243+0000","last_peered":"2023-07-14T11:11:09.290243+0000","last_clean":"2023-07-14T11:11:09.290243+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:11:09.290243+0000","last_undegraded":"2023-07-14T11:11:09.290243+0000","last_fullsized":"2023-07-14T11:11:09.290243+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:34.169535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719060+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:11:08.719060+0000","last_peered":"2023-07-14T11:11:08.719060+0000","last_clean":"2023-07-14T11:11:08.719060+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:11:08.719060+0000","last_undegraded":"2023-07-14T11:11:08.719060+0000","last_fullsized":"2023-07-14T11:11:08.719060+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:24.691976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290657+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:11:09.290657+0000","last_peered":"2023-07-14T11:11:09.290657+0000","last_clean":"2023-07-14T11:11:09.290657+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:11:09.290657+0000","last_undegraded":"2023-07-14T11:11:09.290657+0000","last_fullsized":"2023-07-14T11:11:09.290657+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:53:37.211874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720108+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:11:08.720108+0000","last_peered":"2023-07-14T11:11:08.720108+0000","last_clean":"2023-07-14T11:11:08.720108+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:11:08.720108+0000","last_undegraded":"2023-07-14T11:11:08.720108+0000","last_fullsized":"2023-07-14T11:11:08.720108+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:55:08.685860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289465+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:11:09.289465+0000","last_peered":"2023-07-14T11:11:09.289465+0000","last_clean":"2023-07-14T11:11:09.289465+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:11:09.289465+0000","last_undegraded":"2023-07-14T11:11:09.289465+0000","last_fullsized":"2023-07-14T11:11:09.289465+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:07:02.366836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290356+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:11:09.290356+0000","last_peered":"2023-07-14T11:11:09.290356+0000","last_clean":"2023-07-14T11:11:09.290356+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:11:09.290356+0000","last_undegraded":"2023-07-14T11:11:09.290356+0000","last_fullsized":"2023-07-14T11:11:09.290356+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:04:23.808018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.521863+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:11:50.521863+0000","last_peered":"2023-07-14T11:11:50.521863+0000","last_clean":"2023-07-14T11:11:50.521863+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:11:50.521863+0000","last_undegraded":"2023-07-14T11:11:50.521863+0000","last_fullsized":"2023-07-14T11:11:50.521863+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:08.806584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289179+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:11:09.289179+0000","last_peered":"2023-07-14T11:11:09.289179+0000","last_clean":"2023-07-14T11:11:09.289179+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:11:09.289179+0000","last_undegraded":"2023-07-14T11:11:09.289179+0000","last_fullsized":"2023-07-14T11:11:09.289179+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:00.991168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719386+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:11:08.719386+0000","last_peered":"2023-07-14T11:11:08.719386+0000","last_clean":"2023-07-14T11:11:08.719386+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:11:08.719386+0000","last_undegraded":"2023-07-14T11:11:08.719386+0000","last_fullsized":"2023-07-14T11:11:08.719386+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:10:03.730236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722670+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:11:08.722670+0000","last_peered":"2023-07-14T11:11:08.722670+0000","last_clean":"2023-07-14T11:11:08.722670+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:11:08.722670+0000","last_undegraded":"2023-07-14T11:11:08.722670+0000","last_fullsized":"2023-07-14T11:11:08.722670+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:46:28.743826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719745+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:11:08.719745+0000","last_peered":"2023-07-14T11:11:08.719745+0000","last_clean":"2023-07-14T11:11:08.719745+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:11:08.719745+0000","last_undegraded":"2023-07-14T11:11:08.719745+0000","last_fullsized":"2023-07-14T11:11:08.719745+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:26:00.226723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719524+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:11:08.719524+0000","last_peered":"2023-07-14T11:11:08.719524+0000","last_clean":"2023-07-14T11:11:08.719524+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:11:08.719524+0000","last_undegraded":"2023-07-14T11:11:08.719524+0000","last_fullsized":"2023-07-14T11:11:08.719524+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:30.579895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723335+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:11:08.723335+0000","last_peered":"2023-07-14T11:11:08.723335+0000","last_clean":"2023-07-14T11:11:08.723335+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:11:08.723335+0000","last_undegraded":"2023-07-14T11:11:08.723335+0000","last_fullsized":"2023-07-14T11:11:08.723335+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:23.398441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291267+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:11:09.291267+0000","last_peered":"2023-07-14T11:11:09.291267+0000","last_clean":"2023-07-14T11:11:09.291267+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:11:09.291267+0000","last_undegraded":"2023-07-14T11:11:09.291267+0000","last_fullsized":"2023-07-14T11:11:09.291267+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:13:20.237238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291884+0000","last_change":"2023-07-14T11:11:09.291884+0000","last_active":"2023-07-14T11:11:09.291884+0000","last_peered":"2023-07-14T11:11:09.291884+0000","last_clean":"2023-07-14T11:11:09.291884+0000","last_became_active":"2023-07-14T11:11:09.291383+0000","last_became_peered":"2023-07-14T11:11:09.291383+0000","last_unstale":"2023-07-14T11:11:09.291884+0000","last_undegraded":"2023-07-14T11:11:09.291884+0000","last_fullsized":"2023-07-14T11:11:09.291884+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:58:27.329172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,1],"acting":[8,10,11,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"90'1","reported_seq":32,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290030+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:11:09.290030+0000","last_peered":"2023-07-14T11:11:09.290030+0000","last_clean":"2023-07-14T11:11:09.290030+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:11:09.290030+0000","last_undegraded":"2023-07-14T11:11:09.290030+0000","last_fullsized":"2023-07-14T11:11:09.290030+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:49:08.911423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721739+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:11:08.721739+0000","last_peered":"2023-07-14T11:11:08.721739+0000","last_clean":"2023-07-14T11:11:08.721739+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:08.721739+0000","last_undegraded":"2023-07-14T11:11:08.721739+0000","last_fullsized":"2023-07-14T11:11:08.721739+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:02.329622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719474+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:11:08.719474+0000","last_peered":"2023-07-14T11:11:08.719474+0000","last_clean":"2023-07-14T11:11:08.719474+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:11:08.719474+0000","last_undegraded":"2023-07-14T11:11:08.719474+0000","last_fullsized":"2023-07-14T11:11:08.719474+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:54:56.816899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'5","reported_seq":130,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:49.552173+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:11:49.552173+0000","last_peered":"2023-07-14T11:11:49.552173+0000","last_clean":"2023-07-14T11:11:49.552173+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:11:49.552173+0000","last_undegraded":"2023-07-14T11:11:49.552173+0000","last_fullsized":"2023-07-14T11:11:49.552173+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:42.890747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":94,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721512+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:11:08.721512+0000","last_peered":"2023-07-14T11:11:08.721512+0000","last_clean":"2023-07-14T11:11:08.721512+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:11:08.721512+0000","last_undegraded":"2023-07-14T11:11:08.721512+0000","last_fullsized":"2023-07-14T11:11:08.721512+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.643656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719363+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:11:08.719363+0000","last_peered":"2023-07-14T11:11:08.719363+0000","last_clean":"2023-07-14T11:11:08.719363+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:11:08.719363+0000","last_undegraded":"2023-07-14T11:11:08.719363+0000","last_fullsized":"2023-07-14T11:11:08.719363+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:59:01.913296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719845+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:11:08.719845+0000","last_peered":"2023-07-14T11:11:08.719845+0000","last_clean":"2023-07-14T11:11:08.719845+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:11:08.719845+0000","last_undegraded":"2023-07-14T11:11:08.719845+0000","last_fullsized":"2023-07-14T11:11:08.719845+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:21:32.430525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'5","reported_seq":134,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:52.523992+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:11:52.523992+0000","last_peered":"2023-07-14T11:11:52.523992+0000","last_clean":"2023-07-14T11:11:52.523992+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:11:52.523992+0000","last_undegraded":"2023-07-14T11:11:52.523992+0000","last_fullsized":"2023-07-14T11:11:52.523992+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:37:13.796130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":98,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289479+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:11:09.289479+0000","last_peered":"2023-07-14T11:11:09.289479+0000","last_clean":"2023-07-14T11:11:09.289479+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:11:09.289479+0000","last_undegraded":"2023-07-14T11:11:09.289479+0000","last_fullsized":"2023-07-14T11:11:09.289479+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:41:35.094180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719797+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:11:08.719797+0000","last_peered":"2023-07-14T11:11:08.719797+0000","last_clean":"2023-07-14T11:11:08.719797+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:11:08.719797+0000","last_undegraded":"2023-07-14T11:11:08.719797+0000","last_fullsized":"2023-07-14T11:11:08.719797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:19:31.917453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":40,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417870+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:11:09.417870+0000","last_peered":"2023-07-14T11:11:09.417870+0000","last_clean":"2023-07-14T11:11:09.417870+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:11:09.417870+0000","last_undegraded":"2023-07-14T11:11:09.417870+0000","last_fullsized":"2023-07-14T11:11:09.417870+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:36:12.795737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":359,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289040+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:11:09.289040+0000","last_peered":"2023-07-14T11:11:09.289040+0000","last_clean":"2023-07-14T11:11:09.289040+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:11:09.289040+0000","last_undegraded":"2023-07-14T11:11:09.289040+0000","last_fullsized":"2023-07-14T11:11:09.289040+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:28:25.142388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720162+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:11:08.720162+0000","last_peered":"2023-07-14T11:11:08.720162+0000","last_clean":"2023-07-14T11:11:08.720162+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:11:08.720162+0000","last_undegraded":"2023-07-14T11:11:08.720162+0000","last_fullsized":"2023-07-14T11:11:08.720162+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:39:21.892266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718829+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:11:08.718829+0000","last_peered":"2023-07-14T11:11:08.718829+0000","last_clean":"2023-07-14T11:11:08.718829+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:11:08.718829+0000","last_undegraded":"2023-07-14T11:11:08.718829+0000","last_fullsized":"2023-07-14T11:11:08.718829+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:43:18.820300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'4","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289160+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:11:09.289160+0000","last_peered":"2023-07-14T11:11:09.289160+0000","last_clean":"2023-07-14T11:11:09.289160+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:11:09.289160+0000","last_undegraded":"2023-07-14T11:11:09.289160+0000","last_fullsized":"2023-07-14T11:11:09.289160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:59.013859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289968+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:11:09.289968+0000","last_peered":"2023-07-14T11:11:09.289968+0000","last_clean":"2023-07-14T11:11:09.289968+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:11:09.289968+0000","last_undegraded":"2023-07-14T11:11:09.289968+0000","last_fullsized":"2023-07-14T11:11:09.289968+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:29:24.636531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723306+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:11:08.723306+0000","last_peered":"2023-07-14T11:11:08.723306+0000","last_clean":"2023-07-14T11:11:08.723306+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:11:08.723306+0000","last_undegraded":"2023-07-14T11:11:08.723306+0000","last_fullsized":"2023-07-14T11:11:08.723306+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:37.536988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719233+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:11:08.719233+0000","last_peered":"2023-07-14T11:11:08.719233+0000","last_clean":"2023-07-14T11:11:08.719233+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:11:08.719233+0000","last_undegraded":"2023-07-14T11:11:08.719233+0000","last_fullsized":"2023-07-14T11:11:08.719233+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:03.090653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719888+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:11:08.719888+0000","last_peered":"2023-07-14T11:11:08.719888+0000","last_clean":"2023-07-14T11:11:08.719888+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:11:08.719888+0000","last_undegraded":"2023-07-14T11:11:08.719888+0000","last_fullsized":"2023-07-14T11:11:08.719888+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:53.362733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"90'7","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290126+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:11:09.290126+0000","last_peered":"2023-07-14T11:11:09.290126+0000","last_clean":"2023-07-14T11:11:09.290126+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:11:09.290126+0000","last_undegraded":"2023-07-14T11:11:09.290126+0000","last_fullsized":"2023-07-14T11:11:09.290126+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:27:46.858212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720504+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:11:08.720504+0000","last_peered":"2023-07-14T11:11:08.720504+0000","last_clean":"2023-07-14T11:11:08.720504+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:11:08.720504+0000","last_undegraded":"2023-07-14T11:11:08.720504+0000","last_fullsized":"2023-07-14T11:11:08.720504+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:05.284617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290123+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:11:09.290123+0000","last_peered":"2023-07-14T11:11:09.290123+0000","last_clean":"2023-07-14T11:11:09.290123+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:11:09.290123+0000","last_undegraded":"2023-07-14T11:11:09.290123+0000","last_fullsized":"2023-07-14T11:11:09.290123+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:02:54.462119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418279+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:11:09.418279+0000","last_peered":"2023-07-14T11:11:09.418279+0000","last_clean":"2023-07-14T11:11:09.418279+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:11:09.418279+0000","last_undegraded":"2023-07-14T11:11:09.418279+0000","last_fullsized":"2023-07-14T11:11:09.418279+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:28:55.029604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289355+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:11:09.289355+0000","last_peered":"2023-07-14T11:11:09.289355+0000","last_clean":"2023-07-14T11:11:09.289355+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:11:09.289355+0000","last_undegraded":"2023-07-14T11:11:09.289355+0000","last_fullsized":"2023-07-14T11:11:09.289355+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:35:10.693230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418030+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:11:09.418030+0000","last_peered":"2023-07-14T11:11:09.418030+0000","last_clean":"2023-07-14T11:11:09.418030+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:11:09.418030+0000","last_undegraded":"2023-07-14T11:11:09.418030+0000","last_fullsized":"2023-07-14T11:11:09.418030+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:44:16.733738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000828+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:11:09.000828+0000","last_peered":"2023-07-14T11:11:09.000828+0000","last_clean":"2023-07-14T11:11:09.000828+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:11:09.000828+0000","last_undegraded":"2023-07-14T11:11:09.000828+0000","last_fullsized":"2023-07-14T11:11:09.000828+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:33:19.700027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722662+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:11:08.722662+0000","last_peered":"2023-07-14T11:11:08.722662+0000","last_clean":"2023-07-14T11:11:08.722662+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:11:08.722662+0000","last_undegraded":"2023-07-14T11:11:08.722662+0000","last_fullsized":"2023-07-14T11:11:08.722662+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:14:20.212946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290230+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:11:09.290230+0000","last_peered":"2023-07-14T11:11:09.290230+0000","last_clean":"2023-07-14T11:11:09.290230+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:11:09.290230+0000","last_undegraded":"2023-07-14T11:11:09.290230+0000","last_fullsized":"2023-07-14T11:11:09.290230+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:02:32.312663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"90'7","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720199+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:11:08.720199+0000","last_peered":"2023-07-14T11:11:08.720199+0000","last_clean":"2023-07-14T11:11:08.720199+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:11:08.720199+0000","last_undegraded":"2023-07-14T11:11:08.720199+0000","last_fullsized":"2023-07-14T11:11:08.720199+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:55.458801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289387+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:11:09.289387+0000","last_peered":"2023-07-14T11:11:09.289387+0000","last_clean":"2023-07-14T11:11:09.289387+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:11:09.289387+0000","last_undegraded":"2023-07-14T11:11:09.289387+0000","last_fullsized":"2023-07-14T11:11:09.289387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:44.877318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417886+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:11:09.417886+0000","last_peered":"2023-07-14T11:11:09.417886+0000","last_clean":"2023-07-14T11:11:09.417886+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:11:09.417886+0000","last_undegraded":"2023-07-14T11:11:09.417886+0000","last_fullsized":"2023-07-14T11:11:09.417886+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:22:03.929374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290180+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:11:09.290180+0000","last_peered":"2023-07-14T11:11:09.290180+0000","last_clean":"2023-07-14T11:11:09.290180+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:11:09.290180+0000","last_undegraded":"2023-07-14T11:11:09.290180+0000","last_fullsized":"2023-07-14T11:11:09.290180+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:09.530818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720181+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:11:08.720181+0000","last_peered":"2023-07-14T11:11:08.720181+0000","last_clean":"2023-07-14T11:11:08.720181+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:11:08.720181+0000","last_undegraded":"2023-07-14T11:11:08.720181+0000","last_fullsized":"2023-07-14T11:11:08.720181+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:28:36.511332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417990+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:11:09.417990+0000","last_peered":"2023-07-14T11:11:09.417990+0000","last_clean":"2023-07-14T11:11:09.417990+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:11:09.417990+0000","last_undegraded":"2023-07-14T11:11:09.417990+0000","last_fullsized":"2023-07-14T11:11:09.417990+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:34:29.312985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":35,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722793+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:11:08.722793+0000","last_peered":"2023-07-14T11:11:08.722793+0000","last_clean":"2023-07-14T11:11:08.722793+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:11:08.722793+0000","last_undegraded":"2023-07-14T11:11:08.722793+0000","last_fullsized":"2023-07-14T11:11:08.722793+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:34:50.906895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289355+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:11:09.289355+0000","last_peered":"2023-07-14T11:11:09.289355+0000","last_clean":"2023-07-14T11:11:09.289355+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:11:09.289355+0000","last_undegraded":"2023-07-14T11:11:09.289355+0000","last_fullsized":"2023-07-14T11:11:09.289355+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:27:25.782530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000947+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:11:09.000947+0000","last_peered":"2023-07-14T11:11:09.000947+0000","last_clean":"2023-07-14T11:11:09.000947+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:11:09.000947+0000","last_undegraded":"2023-07-14T11:11:09.000947+0000","last_fullsized":"2023-07-14T11:11:09.000947+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:31:40.967816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.566007+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:11:50.566007+0000","last_peered":"2023-07-14T11:11:50.566007+0000","last_clean":"2023-07-14T11:11:50.566007+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:11:50.566007+0000","last_undegraded":"2023-07-14T11:11:50.566007+0000","last_fullsized":"2023-07-14T11:11:50.566007+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:28.815042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1122,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":97,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289850+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:11:09.289850+0000","last_peered":"2023-07-14T11:11:09.289850+0000","last_clean":"2023-07-14T11:11:09.289850+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:11:09.289850+0000","last_undegraded":"2023-07-14T11:11:09.289850+0000","last_fullsized":"2023-07-14T11:11:09.289850+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:30:10.951768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288876+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:11:09.288876+0000","last_peered":"2023-07-14T11:11:09.288876+0000","last_clean":"2023-07-14T11:11:09.288876+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:11:09.288876+0000","last_undegraded":"2023-07-14T11:11:09.288876+0000","last_fullsized":"2023-07-14T11:11:09.288876+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:35.079631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720056+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:11:08.720056+0000","last_peered":"2023-07-14T11:11:08.720056+0000","last_clean":"2023-07-14T11:11:08.720056+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:11:08.720056+0000","last_undegraded":"2023-07-14T11:11:08.720056+0000","last_fullsized":"2023-07-14T11:11:08.720056+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:24:42.663850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289490+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:11:09.289490+0000","last_peered":"2023-07-14T11:11:09.289490+0000","last_clean":"2023-07-14T11:11:09.289490+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:11:09.289490+0000","last_undegraded":"2023-07-14T11:11:09.289490+0000","last_fullsized":"2023-07-14T11:11:09.289490+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:58.994547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719226+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:11:08.719226+0000","last_peered":"2023-07-14T11:11:08.719226+0000","last_clean":"2023-07-14T11:11:08.719226+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:11:08.719226+0000","last_undegraded":"2023-07-14T11:11:08.719226+0000","last_fullsized":"2023-07-14T11:11:08.719226+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:14.394847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"90'5","reported_seq":45,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719234+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:11:08.719234+0000","last_peered":"2023-07-14T11:11:08.719234+0000","last_clean":"2023-07-14T11:11:08.719234+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:11:08.719234+0000","last_undegraded":"2023-07-14T11:11:08.719234+0000","last_fullsized":"2023-07-14T11:11:08.719234+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:47:15.179813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289653+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:11:09.289653+0000","last_peered":"2023-07-14T11:11:09.289653+0000","last_clean":"2023-07-14T11:11:09.289653+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:11:09.289653+0000","last_undegraded":"2023-07-14T11:11:09.289653+0000","last_fullsized":"2023-07-14T11:11:09.289653+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:11:51.768155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289543+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:11:09.289543+0000","last_peered":"2023-07-14T11:11:09.289543+0000","last_clean":"2023-07-14T11:11:09.289543+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:11:09.289543+0000","last_undegraded":"2023-07-14T11:11:09.289543+0000","last_fullsized":"2023-07-14T11:11:09.289543+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:14.076462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290470+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:11:09.290470+0000","last_peered":"2023-07-14T11:11:09.290470+0000","last_clean":"2023-07-14T11:11:09.290470+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:11:09.290470+0000","last_undegraded":"2023-07-14T11:11:09.290470+0000","last_fullsized":"2023-07-14T11:11:09.290470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:22:58.418691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290343+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:11:09.290343+0000","last_peered":"2023-07-14T11:11:09.290343+0000","last_clean":"2023-07-14T11:11:09.290343+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:11:09.290343+0000","last_undegraded":"2023-07-14T11:11:09.290343+0000","last_fullsized":"2023-07-14T11:11:09.290343+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:07:35.545905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289064+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:11:09.289064+0000","last_peered":"2023-07-14T11:11:09.289064+0000","last_clean":"2023-07-14T11:11:09.289064+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:11:09.289064+0000","last_undegraded":"2023-07-14T11:11:09.289064+0000","last_fullsized":"2023-07-14T11:11:09.289064+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:45:00.837634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":35,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719320+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:11:08.719320+0000","last_peered":"2023-07-14T11:11:08.719320+0000","last_clean":"2023-07-14T11:11:08.719320+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:11:08.719320+0000","last_undegraded":"2023-07-14T11:11:08.719320+0000","last_fullsized":"2023-07-14T11:11:08.719320+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:36.474008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291336+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:11:09.291336+0000","last_peered":"2023-07-14T11:11:09.291336+0000","last_clean":"2023-07-14T11:11:09.291336+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:11:09.291336+0000","last_undegraded":"2023-07-14T11:11:09.291336+0000","last_fullsized":"2023-07-14T11:11:09.291336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:26.293702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290764+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:11:09.290764+0000","last_peered":"2023-07-14T11:11:09.290764+0000","last_clean":"2023-07-14T11:11:09.290764+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:11:09.290764+0000","last_undegraded":"2023-07-14T11:11:09.290764+0000","last_fullsized":"2023-07-14T11:11:09.290764+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:45.932302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":15,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290111+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:11:09.290111+0000","last_peered":"2023-07-14T11:11:09.290111+0000","last_clean":"2023-07-14T11:11:09.290111+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:11:09.290111+0000","last_undegraded":"2023-07-14T11:11:09.290111+0000","last_fullsized":"2023-07-14T11:11:09.290111+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:00.414644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719489+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:11:08.719489+0000","last_peered":"2023-07-14T11:11:08.719489+0000","last_clean":"2023-07-14T11:11:08.719489+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:11:08.719489+0000","last_undegraded":"2023-07-14T11:11:08.719489+0000","last_fullsized":"2023-07-14T11:11:08.719489+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:03.664035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720295+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:11:08.720295+0000","last_peered":"2023-07-14T11:11:08.720295+0000","last_clean":"2023-07-14T11:11:08.720295+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:11:08.720295+0000","last_undegraded":"2023-07-14T11:11:08.720295+0000","last_fullsized":"2023-07-14T11:11:08.720295+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:05:07.837607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001328+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:11:09.001328+0000","last_peered":"2023-07-14T11:11:09.001328+0000","last_clean":"2023-07-14T11:11:09.001328+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:11:09.001328+0000","last_undegraded":"2023-07-14T11:11:09.001328+0000","last_fullsized":"2023-07-14T11:11:09.001328+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:01.143943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719197+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:11:08.719197+0000","last_peered":"2023-07-14T11:11:08.719197+0000","last_clean":"2023-07-14T11:11:08.719197+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:11:08.719197+0000","last_undegraded":"2023-07-14T11:11:08.719197+0000","last_fullsized":"2023-07-14T11:11:08.719197+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:14:33.943135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290872+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:11:09.290872+0000","last_peered":"2023-07-14T11:11:09.290872+0000","last_clean":"2023-07-14T11:11:09.290872+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:11:09.290872+0000","last_undegraded":"2023-07-14T11:11:09.290872+0000","last_fullsized":"2023-07-14T11:11:09.290872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:51:23.080257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":10282,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":1004,"num_read_kb":526,"num_write":211,"num_write_kb":69,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":30846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185,"ondisk_log_size":185,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416423,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":1020,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":1044480,"data_stored":509181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54800000000000004,"15min":0.55600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.439,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83099999999999996,"5min":1.202,"15min":1.202},"last":0.61899999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64400000000000002,"15min":0.64100000000000001},"min":{"1min":0.52900000000000003,"5min":0.443,"15min":0.443},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.63100000000000001,"15min":0.627},"min":{"1min":0.49099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73499999999999999,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.379},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.35399999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.74199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.67100000000000004,"15min":0.67500000000000004},"min":{"1min":0.496,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81299999999999994,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.68300000000000005,"15min":0.68100000000000005},"min":{"1min":0.56200000000000006,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85099999999999998,"5min":1.05,"15min":1.05},"last":0.70399999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79200000000000004,"5min":1.254,"15min":1.254},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.61099999999999999,"15min":0.61299999999999999},"min":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84599999999999997,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71199999999999997,"15min":0.71299999999999997},"min":{"1min":0.495,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73799999999999999,"15min":0.74099999999999999},"min":{"1min":0.53600000000000003,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.92900000000000005,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.72799999999999998,"15min":0.72599999999999998},"min":{"1min":0.59699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.91400000000000003,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.57499999999999996,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.89500000000000002,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.80400000000000005}]},{"osd":8,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.754},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.53700000000000003,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84099999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78600000000000003,"15min":0.78700000000000003},"min":{"1min":0.61799999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":0.95299999999999996,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.80400000000000005,"15min":0.80200000000000005},"min":{"1min":0.58499999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.76900000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.82499999999999996,"15min":0.82199999999999995},"min":{"1min":0.64800000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0149999999999999,"5min":1.411,"15min":1.411},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.66700000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.99299999999999999,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.82499999999999996}]}]},{"osd":10,"up_from":72,"seq":309237645354,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28596,"kb_used_data":1644,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716556,"statfs":{"total":95995035648,"available":95965753344,"internally_reserved":0,"allocated":1683456,"data_stored":1104145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.82799999999999996,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.77700000000000002,"15min":0.79500000000000004},"min":{"1min":0.54500000000000004,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90500000000000003,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.46000000000000002,"15min":0.45600000000000002},"min":{"1min":0.33100000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59099999999999997,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.85399999999999998,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.436}]},{"osd":2,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.65800000000000003,"15min":0.64400000000000002},"min":{"1min":0.47199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91400000000000003,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.66300000000000003,"15min":0.66700000000000004},"min":{"1min":0.44400000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80000000000000004,"5min":1.796,"15min":1.796},"last":0.52100000000000002}]},{"osd":3,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.80700000000000005,"15min":0.88100000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69599999999999995,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.66500000000000004,"15min":0.67300000000000004},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85199999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.60699999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85499999999999998,"5min":1.839,"15min":1.839},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.57599999999999996,"15min":0.56200000000000006},"min":{"1min":0.39900000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81000000000000005,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.53500000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.73399999999999999,"15min":0.72799999999999998},"min":{"1min":0.60099999999999998,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.90900000000000003,"5min":1.165,"15min":1.165},"last":0.624},{"interface":"front","average":{"1min":0.746,"5min":0.76400000000000001,"15min":0.76900000000000002},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":0.90100000000000002,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.74199999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.77100000000000002,"15min":0.77900000000000003},"min":{"1min":0.621,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.88100000000000001,"5min":1.897,"15min":1.897},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.747,"5min":0.876,"15min":0.93000000000000005},"min":{"1min":0.55500000000000005,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96499999999999997,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.72699999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.78600000000000003,"15min":0.78400000000000003},"min":{"1min":0.63700000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.02,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.83699999999999997,"15min":0.84999999999999998},"min":{"1min":0.58699999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.999,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.69999999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.84199999999999997,"15min":0.84599999999999997},"min":{"1min":0.64100000000000001,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0169999999999999,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.871,"15min":0.88700000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.98799999999999999,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.67200000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.85299999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":0.98699999999999999,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.99199999999999999,"15min":1.0189999999999999},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.048,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":0.81399999999999995}]}]},{"osd":9,"up_from":67,"seq":287762808878,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.504,"15min":0.52600000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72299999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.59099999999999997,"15min":0.621},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.98199999999999998,"5min":1.488,"15min":1.488},"last":0.38500000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.75,"15min":0.76500000000000001},"min":{"1min":0.48799999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.302,"5min":1.754,"15min":1.754},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69399999999999995,"15min":0.70599999999999996},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.55500000000000005}]},{"osd":2,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.72899999999999998,"15min":0.75600000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.103,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.73799999999999999,"15min":0.73999999999999999},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.3109999999999999,"5min":5.3109999999999999,"15min":5.3109999999999999},"last":0.62},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.68000000000000005,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.751,"15min":0.755},"min":{"1min":0.59099999999999997,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.214,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.81100000000000005,"15min":0.81699999999999995},"min":{"1min":0.64500000000000002,"5min":0.498,"15min":0.498},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83099999999999996,"15min":0.86299999999999999},"min":{"1min":0.55100000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.98199999999999998,"5min":1.242,"15min":1.242},"last":0.497},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.80000000000000004,"15min":0.83499999999999996},"min":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.097,"5min":1.258,"15min":1.258},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":0.95799999999999996,"15min":0.95599999999999996},"min":{"1min":0.60799999999999998,"5min":0.495,"15min":0.495},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":5.2089999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92200000000000004,"15min":0.96499999999999997},"min":{"1min":0.621,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.5269999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.71099999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.87,"15min":0.90500000000000003},"min":{"1min":0.60199999999999998,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2609999999999999,"5min":1.331,"15min":1.331},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.95199999999999996,"15min":0.99199999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.196,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.53700000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.875,"15min":0.90400000000000003},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.298,"5min":1.7310000000000001,"15min":1.7310000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.137,"5min":1.002,"15min":0.98599999999999999},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":5.3579999999999997,"5min":5.3579999999999997,"15min":5.3579999999999997},"last":0.68200000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.82799999999999996,"15min":0.82499999999999996},"min":{"1min":0.59599999999999997,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.877,"15min":0.88200000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.64000000000000001}]}]},{"osd":8,"up_from":59,"seq":253403070515,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27992,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717160,"statfs":{"total":95995035648,"available":95966371840,"internally_reserved":0,"allocated":1064960,"data_stored":509537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83899999999999997,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59399999999999997,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.84899999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.55900000000000005,"15min":0.53600000000000003},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78200000000000003,"5min":0.875,"15min":0.875},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.61499999999999999,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79800000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.69599999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72299999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.71599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.40300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93000000000000005,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91800000000000004,"5min":1.079,"15min":1.079},"last":0.68700000000000006}]},{"osd":4,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.61899999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.89700000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.64800000000000002,"15min":0.63600000000000001},"min":{"1min":0.56799999999999995,"5min":0.37,"15min":0.37},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.73699999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.79800000000000004,"5min":1.052,"15min":1.052},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82699999999999996,"5min":1.135,"15min":1.135},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.70599999999999996,"15min":0.70799999999999996},"min":{"1min":0.48899999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90100000000000002,"5min":1.008,"15min":1.008},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.73599999999999999,"15min":0.74199999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.71699999999999997},"min":{"1min":0.58199999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":1.028,"15min":1.028},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.73799999999999999,"15min":0.73199999999999998},"min":{"1min":0.54500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.99399999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.629}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76400000000000001,"15min":0.75600000000000001},"min":{"1min":0.61599999999999999,"5min":0.433,"15min":0.433},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.79100000000000004,"15min":0.77900000000000003},"min":{"1min":0.63100000000000001,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0309999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.81599999999999995}]},{"osd":10,"last update":"Fri Jul 14 11:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.75700000000000001,"15min":0.72899999999999998},"min":{"1min":0.65800000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.753},{"interface":"front","average":{"1min":0.876,"5min":0.78700000000000003,"15min":0.753},"min":{"1min":0.65100000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.081,"5min":1.085,"15min":1.085},"last":0.85399999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.81999999999999995,"15min":0.81399999999999995},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.0569999999999999,"5min":1.117,"15min":1.117},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.79200000000000004,"15min":0.78400000000000003},"min":{"1min":0.67200000000000004,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.83699999999999997}]}]},{"osd":7,"up_from":51,"seq":219043332151,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1616,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1654784,"data_stored":1101130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.66300000000000003,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58599999999999997,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59299999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.64100000000000001,"15min":0.65200000000000002},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.92200000000000004,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":0.872,"15min":0.872},"last":0.625}]},{"osd":3,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.61599999999999999,"15min":0.58799999999999997},"min":{"1min":0.44400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.89200000000000002,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.63500000000000001,"15min":0.60299999999999998},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81899999999999995,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.78700000000000003,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69099999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.76600000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.70599999999999996,"15min":0.72199999999999998},"min":{"1min":0.55000000000000004,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.878,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.56899999999999995,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.88100000000000001,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.69099999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.74299999999999999,"15min":0.73799999999999999},"min":{"1min":0.47499999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.05,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.83199999999999996,"15min":0.75600000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":3.7810000000000001,"5min":7.3719999999999999,"15min":7.3719999999999999},"last":0.86199999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.76100000000000001,"15min":0.75900000000000001},"min":{"1min":0.65200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.96599999999999997,"5min":1,"15min":1},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72199999999999998,"15min":0.71999999999999997},"min":{"1min":0.625,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.746}]},{"osd":9,"last update":"Fri Jul 14 11:11:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.83599999999999997,"15min":0.81799999999999995},"min":{"1min":0.61199999999999999,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":3.7109999999999999,"5min":3.7109999999999999,"15min":3.7109999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.875,"15min":0.81899999999999995},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":7.2489999999999997,"5min":7.2489999999999997,"15min":7.2489999999999997},"last":0.89500000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.72999999999999998,"15min":0.71999999999999997},"min":{"1min":0.56799999999999995,"5min":0.442,"15min":0.442},"max":{"1min":0.91500000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.78900000000000003,"15min":0.79000000000000004},"min":{"1min":0.59999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.0580000000000001,"5min":1.216,"15min":1.216},"last":0.84599999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.83999999999999997,"15min":0.83599999999999997},"min":{"1min":0.70399999999999996,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":0.98599999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.877},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.85599999999999998,"15min":0.84999999999999998},"min":{"1min":0.79100000000000004,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.0129999999999999,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.92000000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495677,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.51000000000000001,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.64000000000000001,"15min":0.61399999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":0.47099999999999997}]},{"osd":1,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.0600000000000001,"5min":1.407,"15min":1.407},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.751}]},{"osd":2,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.73699999999999999},"min":{"1min":0.60099999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.75900000000000001,"15min":0.72099999999999997},"min":{"1min":0.54700000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":4.8250000000000002},"last":0.68600000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.67100000000000004,"15min":0.64300000000000002},"min":{"1min":0.46700000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.623,"15min":0.60399999999999998},"min":{"1min":0.43099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.4540000000000002,"5min":2.4540000000000002,"15min":2.4540000000000002},"last":0.621}]},{"osd":4,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76600000000000001,"15min":0.747},"min":{"1min":0.438,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.028,"5min":1.302,"15min":1.302},"last":0.73499999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.73399999999999999,"15min":0.74199999999999999},"min":{"1min":0.57499999999999996,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.84299999999999997,"15min":0.82399999999999995},"min":{"1min":0.61599999999999999,"5min":0.441,"15min":0.441},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.65600000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.84899999999999998,"15min":0.81000000000000005},"min":{"1min":0.51300000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.83099999999999996,"15min":0.81799999999999995},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.113,"5min":1.502,"15min":1.502},"last":0.97699999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.64800000000000002,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.93400000000000005,"15min":0.96599999999999997},"min":{"1min":0.62,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2270000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.89800000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.98499999999999999,"15min":1.0149999999999999},"min":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":2.577,"5min":2.577,"15min":2.577},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0009999999999999,"15min":1.0329999999999999},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":2.5630000000000002,"5min":2.5630000000000002,"15min":2.5630000000000002},"last":0.879}]},{"osd":10,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.96299999999999997,"15min":0.97899999999999998},"min":{"1min":0.71299999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.101,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.85199999999999998,"15min":0.86799999999999999},"min":{"1min":0.50800000000000001,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.998,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.80000000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:11:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.0349999999999999,"15min":1.012},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":4.6769999999999996,"5min":4.6769999999999996,"15min":4.6769999999999996},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.88300000000000001,"15min":0.88900000000000001},"min":{"1min":0.65300000000000002,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.101,"5min":1.248,"15min":1.248},"last":0.85999999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574933,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":513793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.432,"5min":0.253,"15min":0.253},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.60299999999999998,"15min":0.53100000000000003},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.66500000000000004}]},{"osd":2,"last update":"Fri Jul 14 11:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.66700000000000004,"15min":0.57099999999999995},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89200000000000002,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58999999999999997,"15min":0.52700000000000002},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.33900000000000002},"max":{"1min":0.82699999999999996,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.60299999999999998,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.863,"5min":1.863,"15min":1.863},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.71699999999999997,"15min":0.60899999999999999},"min":{"1min":0.45600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.8420000000000001,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.84699999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.71699999999999997,"15min":0.66100000000000003},"min":{"1min":0.56100000000000005,"5min":0.38,"15min":0.38},"max":{"1min":0.96499999999999997,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64400000000000002,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89600000000000002,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.69299999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.72799999999999998,"15min":0.67900000000000005},"min":{"1min":0.48699999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.4100000000000001,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.875},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.79500000000000004,"15min":0.73899999999999999},"min":{"1min":0.55600000000000005,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":2.0539999999999998,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":1.028}]},{"osd":7,"last update":"Fri Jul 14 11:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.71299999999999997,"15min":0.67400000000000004},"min":{"1min":0.42499999999999999,"5min":0.316,"15min":0.316},"max":{"1min":1.0880000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.65900000000000003,"15min":0.629},"min":{"1min":0.40500000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.94099999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.68999999999999995,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.01,"5min":1.04,"15min":1.04},"last":1.006},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.77800000000000002,"15min":0.72999999999999998},"min":{"1min":0.62,"5min":0.441,"15min":0.441},"max":{"1min":1.159,"5min":1.165,"15min":1.165},"last":0.97299999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.85199999999999998,"15min":0.73699999999999999},"min":{"1min":0.57699999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":6.6980000000000004,"5min":6.6980000000000004,"15min":6.6980000000000004},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.88100000000000001,"15min":0.77700000000000002},"min":{"1min":0.68400000000000005,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":6.593,"5min":6.593,"15min":6.593},"last":1.0549999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.83699999999999997,"15min":0.83199999999999996},"min":{"1min":0.45600000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.056,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.075},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73699999999999999,"15min":0.72099999999999997},"min":{"1min":0.47399999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.0720000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.73799999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.96499999999999997,"15min":0.97799999999999998},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.157,"5min":1.256,"15min":1.256},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.877,"15min":0.88800000000000001},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.83299999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738458,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.69699999999999995,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9039999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66000000000000003,"15min":0.57699999999999996},"min":{"1min":0.42999999999999999,"5min":0.36799999999999999,"15min":0.36499999999999999},"max":{"1min":1.875,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.54000000000000004}]},{"osd":2,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66200000000000003,"15min":0.58499999999999996},"min":{"1min":0.46400000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.93400000000000005,"5min":2.528,"15min":2.528},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66900000000000004,"15min":0.60599999999999998},"min":{"1min":0.44800000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.88,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.76000000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.71299999999999997,"15min":0.73299999999999998},"min":{"1min":0.437,"5min":0.433,"15min":0.433},"max":{"1min":0.82399999999999995,"5min":3.2599999999999998,"15min":3.2599999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.878,"15min":0.91500000000000004},"min":{"1min":0.45500000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.0920000000000001,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.78600000000000003}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.70599999999999996,"15min":0.63600000000000001},"min":{"1min":0.47199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.2829999999999999,"5min":2.48,"15min":2.48},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.80000000000000004,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":5.9610000000000003,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":0.66700000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.81799999999999995,"15min":0.71499999999999997},"min":{"1min":0.55100000000000005,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.4019999999999999,"5min":6.0839999999999996,"15min":6.0839999999999996},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.75800000000000001,"15min":0.67000000000000004},"min":{"1min":0.62,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.02,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":0.95499999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:11:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.77800000000000002,"15min":0.71999999999999997},"min":{"1min":0.433,"5min":0.38,"15min":0.38},"max":{"1min":6.1870000000000003,"5min":6.1870000000000003,"15min":6.1870000000000003},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.74299999999999999,"15min":0.71899999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.7450000000000001,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.79500000000000004,"15min":0.79900000000000004},"min":{"1min":0.42599999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.0580000000000001,"5min":1.323,"15min":1.323},"last":0.62},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.88100000000000001,"15min":0.90100000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91000000000000003,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.68799999999999994}]},{"osd":8,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.91100000000000003,"15min":0.88200000000000001},"min":{"1min":0.55700000000000005,"5min":0.496,"15min":0.496},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.88100000000000001,"15min":0.89100000000000001},"min":{"1min":0.68600000000000005,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.83299999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:11:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.82499999999999996,"15min":0.80500000000000005},"min":{"1min":0.50800000000000001,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.95399999999999996,"15min":0.91800000000000004},"min":{"1min":0.60299999999999998,"5min":0.437,"15min":0.437},"max":{"1min":6.1180000000000003,"5min":6.1180000000000003,"15min":6.1180000000000003},"last":0.93600000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.876,"15min":0.86699999999999999},"min":{"1min":0.57799999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.456,"5min":2.085,"15min":2.085},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.85799999999999998,"15min":0.82799999999999996},"min":{"1min":0.69799999999999995,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.137,"5min":1.359,"15min":1.359},"last":0.92300000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.95499999999999996,"15min":0.90000000000000002},"min":{"1min":0.77200000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":6.0460000000000003,"5min":6.0460000000000003,"15min":6.0460000000000003},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.026,"5min":0.88800000000000001,"15min":0.86499999999999999},"min":{"1min":0.752,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.91200000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28000,"kb_used_data":1048,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717152,"statfs":{"total":95995035648,"available":95966363648,"internally_reserved":0,"allocated":1073152,"data_stored":513692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.872,"5min":1.304,"15min":1.304},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.54200000000000004},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.33100000000000002},"max":{"1min":1.0229999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.44700000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.59099999999999997,"15min":0.54800000000000004},"min":{"1min":0.42499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.875,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.503},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.62,"15min":0.56599999999999995},"min":{"1min":0.54500000000000004,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.61899999999999999,"15min":0.60499999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.70599999999999996,"15min":0.64700000000000002},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":3.2210000000000001,"5min":3.2210000000000001,"15min":3.2210000000000001},"last":0.58199999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.72099999999999997,"15min":0.72599999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.74399999999999999,"15min":0.745},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.97999999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.624}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.46800000000000003},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73499999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.52600000000000002,"15min":0.48399999999999999},"min":{"1min":0.41399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78800000000000003,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Fri Jul 14 11:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.623,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.69199999999999995,"15min":0.64800000000000002},"min":{"1min":0.57199999999999995,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.83099999999999996}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.79400000000000004,"15min":0.79200000000000004},"min":{"1min":0.59699999999999998,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":0.995,"5min":0.998,"15min":0.998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.77200000000000002,"15min":0.77400000000000002},"min":{"1min":0.57699999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.94799999999999995,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.64900000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.60399999999999998,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.61499999999999999,"15min":0.58899999999999997},"min":{"1min":0.51800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.69499999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:11:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.76400000000000001,"15min":0.75},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.075,"5min":0.85699999999999998,"15min":0.81899999999999995},"min":{"1min":0.70299999999999996,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":3.27,"5min":3.27,"15min":3.27},"last":0.96299999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.80800000000000005,"15min":0.78500000000000003},"min":{"1min":0.64900000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0780000000000001,"5min":1.085,"15min":1.085},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.77100000000000002,"15min":0.74299999999999999},"min":{"1min":0.65600000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.879}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84299999999999997,"15min":0.84199999999999997},"min":{"1min":0.63300000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79000000000000004,"15min":0.78600000000000003},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.84999999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182474,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":508194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.48399999999999999,"15min":0.42099999999999999},"min":{"1min":0.379,"5min":0.252,"15min":0.252},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.44600000000000001},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.3120000000000001,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.48299999999999998}]},{"osd":1,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61599999999999999,"15min":0.59399999999999997},"min":{"1min":0.374,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.69299999999999995,"15min":0.60899999999999999},"min":{"1min":0.57799999999999996,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89400000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.59099999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.67300000000000004,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3879999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.68899999999999995,"15min":0.65100000000000002},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.56899999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.72799999999999998,"15min":0.65300000000000002},"min":{"1min":0.434,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.87},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.77200000000000002,"15min":0.69999999999999996},"min":{"1min":0.61399999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98199999999999998,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.877}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.75900000000000001,"15min":0.73899999999999999},"min":{"1min":0.56499999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.92500000000000004,"5min":1.262,"15min":1.262},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.754,"15min":0.75},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1200000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.622}]},{"osd":6,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.748,"15min":0.72899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.044,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.755,"5min":0.75,"15min":0.75900000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.60699999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.81699999999999995,"15min":0.82699999999999996},"min":{"1min":0.53600000000000003,"5min":0.496,"15min":0.496},"max":{"1min":1.4670000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.83199999999999996,"15min":0.81599999999999995},"min":{"1min":0.53100000000000003,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.84499999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.89000000000000001,"15min":0.90800000000000003},"min":{"1min":0.46500000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.879,"5min":1.879,"15min":1.879},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.95299999999999996,"15min":0.98499999999999999},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.7490000000000001,"5min":1.7490000000000001,"15min":1.7490000000000001},"last":0.78600000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.98799999999999999,"15min":1.02},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.23,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.94299999999999995,"15min":0.96599999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.0009999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.85299999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.85799999999999998,"15min":0.86499999999999999},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.296,"5min":1.351,"15min":1.351},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.874,"5min":0.88200000000000001,"15min":0.88400000000000001},"min":{"1min":0.72399999999999998,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.014,"5min":1.163,"15min":1.163},"last":0.73799999999999999}]}]},{"osd":4,"up_from":33,"seq":141733920839,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39100000000000001,"15min":0.36799999999999999},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.47699999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.39200000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.59099999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52100000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89400000000000002,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.64200000000000002,"15min":0.58899999999999997},"min":{"1min":0.47999999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.874,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63300000000000001,"15min":0.59099999999999997},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.841,"5min":1.841,"15min":1.841},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.622,"15min":0.59199999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.857,"5min":1.857,"15min":1.857},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.64300000000000002,"15min":0.59599999999999997},"min":{"1min":0.46999999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69499999999999995,"15min":0.64600000000000002},"min":{"1min":0.60099999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.95199999999999996,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.63400000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.65300000000000002,"15min":0.63500000000000001},"min":{"1min":0.436,"5min":0.432,"15min":0.432},"max":{"1min":0.871,"5min":1.105,"15min":1.105},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71499999999999997,"15min":0.68200000000000005},"min":{"1min":0.49199999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97599999999999998,"5min":1.137,"15min":1.137},"last":0.76600000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.65400000000000003,"15min":0.61799999999999999},"min":{"1min":0.45000000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.99299999999999999,"5min":1.087,"15min":1.087},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.628,"15min":0.59799999999999998},"min":{"1min":0.47599999999999998,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.82299999999999995,"5min":1.034,"15min":1.034},"last":0.83699999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.73199999999999998,"15min":0.72799999999999998},"min":{"1min":0.54900000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.81000000000000005,"15min":0.80100000000000005},"min":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0329999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.877}]},{"osd":9,"last update":"Fri Jul 14 11:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.81499999999999995,"15min":0.84099999999999997},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":0.95299999999999996,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.81200000000000006,"15min":0.82399999999999995},"min":{"1min":0.64300000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0589999999999999,"5min":1.123,"15min":1.123},"last":0.79600000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86499999999999999,"15min":0.88300000000000001},"min":{"1min":0.65200000000000002,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.075,"5min":1.212,"15min":1.212},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.752,"5min":0.79100000000000004,"15min":0.80100000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93100000000000005,"5min":1.073,"15min":1.073},"last":0.90100000000000002}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.72099999999999997,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86799999999999999,"15min":0.86599999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.97999999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757314,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1667072,"data_stored":1100904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.40500000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.746,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54200000000000004,"15min":0.55400000000000005},"min":{"1min":0.36699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.75700000000000001,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52800000000000002}]},{"osd":1,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59099999999999997,"15min":0.58499999999999996},"min":{"1min":0.42699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.878,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.749},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61799999999999999,"15min":0.59599999999999997},"min":{"1min":0.53000000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83799999999999997,"5min":1.466,"15min":1.466},"last":0.64700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.34300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68999999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.45200000000000001,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60499999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.60399999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.60099999999999998},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41799999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78600000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.69299999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.71299999999999997,"15min":0.65900000000000003},"min":{"1min":0.49199999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88300000000000001,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.69599999999999995,"15min":0.64600000000000002},"min":{"1min":0.51700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90900000000000003,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65300000000000002,"15min":0.66300000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82199999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68600000000000005,"15min":0.68999999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.77100000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.93100000000000005,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.745,"5min":0.73599999999999999,"15min":0.73999999999999999},"min":{"1min":0.54900000000000004,"5min":0.496,"15min":0.496},"max":{"1min":0.93200000000000005,"5min":1.528,"15min":1.528},"last":0.91900000000000004}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.878,"15min":0.878},"last":0.90500000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75600000000000001,"15min":0.74399999999999999},"min":{"1min":0.49299999999999999,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.77200000000000002,"15min":0.76000000000000001},"min":{"1min":0.59199999999999997,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.88400000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.75600000000000001,"15min":0.73599999999999999},"min":{"1min":0.61899999999999999,"5min":0.439,"15min":0.439},"max":{"1min":0.95299999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.91000000000000003,"15min":0.96199999999999997},"min":{"1min":0.59499999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.93100000000000005,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.85099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":2772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":2998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":5740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:11:54.855 INFO:tasks.fwd_scrub.ceph_manager:clean! 2023-07-14T11:11:54.855 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2023-07-14T11:11:55.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:54 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3204742463' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:55.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:54 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3204742463' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:55.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:54 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3204742463' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 -- 172.21.15.40:0/981905315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00100cf0 msgr2=0x7f1d00101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 --2- 172.21.15.40:0/981905315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00100cf0 0x7f1d00101140 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f1cec009990 tx=0x7f1cec009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 -- 172.21.15.40:0/981905315 shutdown_connections 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 --2- 172.21.15.40:0/981905315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00101680 0x7f1d00107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 --2- 172.21.15.40:0/981905315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 0x7f1d000fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 --2- 172.21.15.40:0/981905315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00100cf0 0x7f1d00101140 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 -- 172.21.15.40:0/981905315 >> 172.21.15.40:0/981905315 conn(0x7f1d000fb0e0 msgr2=0x7f1d000fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:55.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 -- 172.21.15.40:0/981905315 shutdown_connections 2023-07-14T11:11:55.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.087+0000 7f1d06bff700 1 -- 172.21.15.40:0/981905315 wait complete. 2023-07-14T11:11:55.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.088+0000 7f1d06bff700 1 Processor -- start 2023-07-14T11:11:55.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.088+0000 7f1d06bff700 1 -- start start 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 0x7f1d000742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 0x7f1d00072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d000735e0 con 0x7f1d00100cf0 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d00073750 con 0x7f1d00101680 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d06bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d0019f810 con 0x7f1d000ffb00 2023-07-14T11:11:55.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55576/0 (socket says 172.21.15.40:55576) 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 -- 172.21.15.40:0/2946258788 learned_addr learned my addr 172.21.15.40:0/2946258788 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d0499b700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 0x7f1d000742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d0519c700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 0x7f1d00072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 -- 172.21.15.40:0/2946258788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 msgr2=0x7f1d000742c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 0x7f1d000742c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 -- 172.21.15.40:0/2946258788 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 msgr2=0x7f1d00072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 0x7f1d00072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 -- 172.21.15.40:0/2946258788 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cec0095f0 con 0x7f1d00100cf0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1d0519c700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 0x7f1d00072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.089+0000 7f1cfffff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f1cec005740 tx=0x7f1cec005ef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.090+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cec01d070 con 0x7f1d00100cf0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.090+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1cec00fbf0 con 0x7f1d00100cf0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.090+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d0019fa90 con 0x7f1d00100cf0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.090+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cec017650 con 0x7f1d00100cf0 2023-07-14T11:11:55.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.090+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d0019ff90 con 0x7f1d00100cf0 2023-07-14T11:11:55.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.091+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d00106ff0 con 0x7f1d00100cf0 2023-07-14T11:11:55.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.093+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f1cec021410 con 0x7f1d00100cf0 2023-07-14T11:11:55.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.094+0000 7f1cfdffb700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 0x7f1cf00a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.094+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f1cec0ce870 con 0x7f1d00100cf0 2023-07-14T11:11:55.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.095+0000 7f1d0499b700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 0x7f1cf00a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.095+0000 7f1d0499b700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 0x7f1cf00a36d0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f1d00100b50 tx=0x7f1cf4009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:55.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.096+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1cec097ca0 con 0x7f1d00100cf0 2023-07-14T11:11:55.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.300+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f1d00060950 con 0x7f1cf00a1220 2023-07-14T11:11:55.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.313+0000 7f1cfdffb700 1 -- 172.21.15.40:0/2946258788 <== mgr.14148 v2:172.21.15.40:6800/2087222828 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+426843 (secure 0 0 0) 0x7f1d00060950 con 0x7f1cf00a1220 2023-07-14T11:11:55.320 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:55.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 msgr2=0x7f1cf00a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 0x7f1cf00a36d0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f1d00100b50 tx=0x7f1cf4009260 comp rx=0 tx=0).stop 2023-07-14T11:11:55.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 msgr2=0x7f1d00072900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 crypto rx=0x7f1cec005740 tx=0x7f1cec005ef0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 shutdown_connections 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f1cf00a1220 0x7f1cf00a36d0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d00101680 0x7f1d00072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d000ffb00 0x7f1d000742c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 --2- 172.21.15.40:0/2946258788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d00100cf0 0x7f1d00072900 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 >> 172.21.15.40:0/2946258788 conn(0x7f1d000fb0e0 msgr2=0x7f1d00104130 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:55.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.318+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 shutdown_connections 2023-07-14T11:11:55.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.319+0000 7f1d06bff700 1 -- 172.21.15.40:0/2946258788 wait complete. 2023-07-14T11:11:55.324 INFO:teuthology.orchestra.run.smithi040.stderr:dumped all 2023-07-14T11:11:55.340 INFO:teuthology.orchestra.run.smithi040.stdout:{"pg_ready":true,"pg_map":{"version":346,"stamp":"2023-07-14T11:11:54.659068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":600650,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":1140,"num_read_kb":645,"num_write":392,"num_write_kb":3007,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":322,"ondisk_log_size":322,"up":419,"acting":419,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":419,"num_osds":12,"num_per_pool_osds":12,"num_per_pool_omap_osds":12,"kb":1124941824,"kb_used":337620,"kb_used_data":14196,"kb_used_omap":0,"kb_used_meta":323328,"kb_avail":1124604204,"statfs":{"total":1151940427776,"available":1151594704896,"internally_reserved":0,"allocated":14536704,"data_stored":7896756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331087872},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":258,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":126,"num_read_kb":71,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002938"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289856+0000","last_change":"2023-07-14T11:09:43.593060+0000","last_active":"2023-07-14T11:11:09.289856+0000","last_peered":"2023-07-14T11:11:09.289856+0000","last_clean":"2023-07-14T11:11:09.289856+0000","last_became_active":"2023-07-14T11:09:43.592707+0000","last_became_peered":"2023-07-14T11:09:43.592707+0000","last_unstale":"2023-07-14T11:11:09.289856+0000","last_undegraded":"2023-07-14T11:11:09.289856+0000","last_fullsized":"2023-07-14T11:11:09.289856+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:21:08.313051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289223+0000","last_change":"2023-07-14T11:09:43.590207+0000","last_active":"2023-07-14T11:11:09.289223+0000","last_peered":"2023-07-14T11:11:09.289223+0000","last_clean":"2023-07-14T11:11:09.289223+0000","last_became_active":"2023-07-14T11:09:43.589862+0000","last_became_peered":"2023-07-14T11:09:43.589862+0000","last_unstale":"2023-07-14T11:11:09.289223+0000","last_undegraded":"2023-07-14T11:11:09.289223+0000","last_fullsized":"2023-07-14T11:11:09.289223+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:31.561921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,11,10],"acting":[0,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289047+0000","last_change":"2023-07-14T11:09:43.595614+0000","last_active":"2023-07-14T11:11:09.289047+0000","last_peered":"2023-07-14T11:11:09.289047+0000","last_clean":"2023-07-14T11:11:09.289047+0000","last_became_active":"2023-07-14T11:09:43.595307+0000","last_became_peered":"2023-07-14T11:09:43.595307+0000","last_unstale":"2023-07-14T11:11:09.289047+0000","last_undegraded":"2023-07-14T11:11:09.289047+0000","last_fullsized":"2023-07-14T11:11:09.289047+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:16:30.208858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418276+0000","last_change":"2023-07-14T11:09:43.592768+0000","last_active":"2023-07-14T11:11:09.418276+0000","last_peered":"2023-07-14T11:11:09.418276+0000","last_clean":"2023-07-14T11:11:09.418276+0000","last_became_active":"2023-07-14T11:09:43.592468+0000","last_became_peered":"2023-07-14T11:09:43.592468+0000","last_unstale":"2023-07-14T11:11:09.418276+0000","last_undegraded":"2023-07-14T11:11:09.418276+0000","last_fullsized":"2023-07-14T11:11:09.418276+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:00:30.994129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,8,2],"acting":[4,8,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722848+0000","last_change":"2023-07-14T11:09:43.596056+0000","last_active":"2023-07-14T11:11:08.722848+0000","last_peered":"2023-07-14T11:11:08.722848+0000","last_clean":"2023-07-14T11:11:08.722848+0000","last_became_active":"2023-07-14T11:09:43.595808+0000","last_became_peered":"2023-07-14T11:09:43.595808+0000","last_unstale":"2023-07-14T11:11:08.722848+0000","last_undegraded":"2023-07-14T11:11:08.722848+0000","last_fullsized":"2023-07-14T11:11:08.722848+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:29:27.534998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,3,6],"acting":[8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722230+0000","last_change":"2023-07-14T11:09:43.588270+0000","last_active":"2023-07-14T11:11:08.722230+0000","last_peered":"2023-07-14T11:11:08.722230+0000","last_clean":"2023-07-14T11:11:08.722230+0000","last_became_active":"2023-07-14T11:09:43.587996+0000","last_became_peered":"2023-07-14T11:09:43.587996+0000","last_unstale":"2023-07-14T11:11:08.722230+0000","last_undegraded":"2023-07-14T11:11:08.722230+0000","last_fullsized":"2023-07-14T11:11:08.722230+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:31:41.760376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,9,4],"acting":[10,9,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289149+0000","last_change":"2023-07-14T11:09:43.599200+0000","last_active":"2023-07-14T11:11:09.289149+0000","last_peered":"2023-07-14T11:11:09.289149+0000","last_clean":"2023-07-14T11:11:09.289149+0000","last_became_active":"2023-07-14T11:09:43.598912+0000","last_became_peered":"2023-07-14T11:09:43.598912+0000","last_unstale":"2023-07-14T11:11:09.289149+0000","last_undegraded":"2023-07-14T11:11:09.289149+0000","last_fullsized":"2023-07-14T11:11:09.289149+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:33.117233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,9,5],"acting":[7,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719836+0000","last_change":"2023-07-14T11:09:43.583657+0000","last_active":"2023-07-14T11:11:08.719836+0000","last_peered":"2023-07-14T11:11:08.719836+0000","last_clean":"2023-07-14T11:11:08.719836+0000","last_became_active":"2023-07-14T11:09:43.582784+0000","last_became_peered":"2023-07-14T11:09:43.582784+0000","last_unstale":"2023-07-14T11:11:08.719836+0000","last_undegraded":"2023-07-14T11:11:08.719836+0000","last_fullsized":"2023-07-14T11:11:08.719836+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:23:33.952593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,10],"acting":[3,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001574+0000","last_change":"2023-07-14T11:09:43.585666+0000","last_active":"2023-07-14T11:11:09.001574+0000","last_peered":"2023-07-14T11:11:09.001574+0000","last_clean":"2023-07-14T11:11:09.001574+0000","last_became_active":"2023-07-14T11:09:43.585315+0000","last_became_peered":"2023-07-14T11:09:43.585315+0000","last_unstale":"2023-07-14T11:11:09.001574+0000","last_undegraded":"2023-07-14T11:11:09.001574+0000","last_fullsized":"2023-07-14T11:11:09.001574+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:31.244279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,11],"acting":[1,5,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721940+0000","last_change":"2023-07-14T11:09:43.593401+0000","last_active":"2023-07-14T11:11:08.721940+0000","last_peered":"2023-07-14T11:11:08.721940+0000","last_clean":"2023-07-14T11:11:08.721940+0000","last_became_active":"2023-07-14T11:09:43.592999+0000","last_became_peered":"2023-07-14T11:09:43.592999+0000","last_unstale":"2023-07-14T11:11:08.721940+0000","last_undegraded":"2023-07-14T11:11:08.721940+0000","last_fullsized":"2023-07-14T11:11:08.721940+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:50:22.375884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,2,4],"acting":[10,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289757+0000","last_change":"2023-07-14T11:09:43.595128+0000","last_active":"2023-07-14T11:11:09.289757+0000","last_peered":"2023-07-14T11:11:09.289757+0000","last_clean":"2023-07-14T11:11:09.289757+0000","last_became_active":"2023-07-14T11:09:43.594436+0000","last_became_peered":"2023-07-14T11:09:43.594436+0000","last_unstale":"2023-07-14T11:11:09.289757+0000","last_undegraded":"2023-07-14T11:11:09.289757+0000","last_fullsized":"2023-07-14T11:11:09.289757+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:49:09.252229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,8,1],"acting":[7,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418209+0000","last_change":"2023-07-14T11:09:43.588820+0000","last_active":"2023-07-14T11:11:09.418209+0000","last_peered":"2023-07-14T11:11:09.418209+0000","last_clean":"2023-07-14T11:11:09.418209+0000","last_became_active":"2023-07-14T11:09:43.585943+0000","last_became_peered":"2023-07-14T11:09:43.585943+0000","last_unstale":"2023-07-14T11:11:09.418209+0000","last_undegraded":"2023-07-14T11:11:09.418209+0000","last_fullsized":"2023-07-14T11:11:09.418209+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:12:39.940446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,8],"acting":[4,10,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719070+0000","last_change":"2023-07-14T11:09:43.585221+0000","last_active":"2023-07-14T11:11:08.719070+0000","last_peered":"2023-07-14T11:11:08.719070+0000","last_clean":"2023-07-14T11:11:08.719070+0000","last_became_active":"2023-07-14T11:09:43.584919+0000","last_became_peered":"2023-07-14T11:09:43.584919+0000","last_unstale":"2023-07-14T11:11:08.719070+0000","last_undegraded":"2023-07-14T11:11:08.719070+0000","last_fullsized":"2023-07-14T11:11:08.719070+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:21:56.387394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,10,5],"acting":[6,10,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288747+0000","last_change":"2023-07-14T11:09:43.596273+0000","last_active":"2023-07-14T11:11:09.288747+0000","last_peered":"2023-07-14T11:11:09.288747+0000","last_clean":"2023-07-14T11:11:09.288747+0000","last_became_active":"2023-07-14T11:09:43.595955+0000","last_became_peered":"2023-07-14T11:09:43.595955+0000","last_unstale":"2023-07-14T11:11:09.288747+0000","last_undegraded":"2023-07-14T11:11:09.288747+0000","last_fullsized":"2023-07-14T11:11:09.288747+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:34.963471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,8,10],"acting":[0,8,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288910+0000","last_change":"2023-07-14T11:09:43.589284+0000","last_active":"2023-07-14T11:11:09.288910+0000","last_peered":"2023-07-14T11:11:09.288910+0000","last_clean":"2023-07-14T11:11:09.288910+0000","last_became_active":"2023-07-14T11:09:43.588956+0000","last_became_peered":"2023-07-14T11:09:43.588956+0000","last_unstale":"2023-07-14T11:11:09.288910+0000","last_undegraded":"2023-07-14T11:11:09.288910+0000","last_fullsized":"2023-07-14T11:11:09.288910+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:27:24.871920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000761+0000","last_change":"2023-07-14T11:09:43.596079+0000","last_active":"2023-07-14T11:11:09.000761+0000","last_peered":"2023-07-14T11:11:09.000761+0000","last_clean":"2023-07-14T11:11:09.000761+0000","last_became_active":"2023-07-14T11:09:43.595741+0000","last_became_peered":"2023-07-14T11:09:43.595741+0000","last_unstale":"2023-07-14T11:11:09.000761+0000","last_undegraded":"2023-07-14T11:11:09.000761+0000","last_fullsized":"2023-07-14T11:11:09.000761+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:22:12.060207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289626+0000","last_change":"2023-07-14T11:09:43.593066+0000","last_active":"2023-07-14T11:11:09.289626+0000","last_peered":"2023-07-14T11:11:09.289626+0000","last_clean":"2023-07-14T11:11:09.289626+0000","last_became_active":"2023-07-14T11:09:43.592706+0000","last_became_peered":"2023-07-14T11:09:43.592706+0000","last_unstale":"2023-07-14T11:11:09.289626+0000","last_undegraded":"2023-07-14T11:11:09.289626+0000","last_fullsized":"2023-07-14T11:11:09.289626+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:33:58.195208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,1,0],"acting":[11,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290140+0000","last_change":"2023-07-14T11:09:43.600945+0000","last_active":"2023-07-14T11:11:09.290140+0000","last_peered":"2023-07-14T11:11:09.290140+0000","last_clean":"2023-07-14T11:11:09.290140+0000","last_became_active":"2023-07-14T11:09:43.600331+0000","last_became_peered":"2023-07-14T11:09:43.600331+0000","last_unstale":"2023-07-14T11:11:09.290140+0000","last_undegraded":"2023-07-14T11:11:09.290140+0000","last_fullsized":"2023-07-14T11:11:09.290140+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:38.328569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,1,10],"acting":[9,1,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290427+0000","last_change":"2023-07-14T11:09:43.596194+0000","last_active":"2023-07-14T11:11:09.290427+0000","last_peered":"2023-07-14T11:11:09.290427+0000","last_clean":"2023-07-14T11:11:09.290427+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:09.290427+0000","last_undegraded":"2023-07-14T11:11:09.290427+0000","last_fullsized":"2023-07-14T11:11:09.290427+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:45:52.717942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290086+0000","last_change":"2023-07-14T11:09:47.614264+0000","last_active":"2023-07-14T11:11:09.290086+0000","last_peered":"2023-07-14T11:11:09.290086+0000","last_clean":"2023-07-14T11:11:09.290086+0000","last_became_active":"2023-07-14T11:09:47.613860+0000","last_became_peered":"2023-07-14T11:09:47.613860+0000","last_unstale":"2023-07-14T11:11:09.290086+0000","last_undegraded":"2023-07-14T11:11:09.290086+0000","last_fullsized":"2023-07-14T11:11:09.290086+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:54:00.281074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,9,0],"acting":[2,3,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1f","version":"90'13","reported_seq":140,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.553095+0000","last_change":"2023-07-14T11:09:42.602158+0000","last_active":"2023-07-14T11:11:50.553095+0000","last_peered":"2023-07-14T11:11:50.553095+0000","last_clean":"2023-07-14T11:11:50.553095+0000","last_became_active":"2023-07-14T11:09:42.601635+0000","last_became_peered":"2023-07-14T11:09:42.601635+0000","last_unstale":"2023-07-14T11:11:50.553095+0000","last_undegraded":"2023-07-14T11:11:50.553095+0000","last_fullsized":"2023-07-14T11:11:50.553095+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":13,"log_dups_size":0,"ondisk_log_size":13,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:46:31.832968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":96,"num_read_kb":96,"num_write":14,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,8],"acting":[9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720370+0000","last_change":"2023-07-14T11:09:43.596768+0000","last_active":"2023-07-14T11:11:08.720370+0000","last_peered":"2023-07-14T11:11:08.720370+0000","last_clean":"2023-07-14T11:11:08.720370+0000","last_became_active":"2023-07-14T11:09:43.596462+0000","last_became_peered":"2023-07-14T11:09:43.596462+0000","last_unstale":"2023-07-14T11:11:08.720370+0000","last_undegraded":"2023-07-14T11:11:08.720370+0000","last_fullsized":"2023-07-14T11:11:08.720370+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:41:19.828306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417507+0000","last_change":"2023-07-14T11:09:47.624851+0000","last_active":"2023-07-14T11:11:09.417507+0000","last_peered":"2023-07-14T11:11:09.417507+0000","last_clean":"2023-07-14T11:11:09.417507+0000","last_became_active":"2023-07-14T11:09:47.624141+0000","last_became_peered":"2023-07-14T11:09:47.624141+0000","last_unstale":"2023-07-14T11:11:09.417507+0000","last_undegraded":"2023-07-14T11:11:09.417507+0000","last_fullsized":"2023-07-14T11:11:09.417507+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:22:30.970594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,9,3],"acting":[4,6,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1e","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720636+0000","last_change":"2023-07-14T11:09:42.600697+0000","last_active":"2023-07-14T11:11:08.720636+0000","last_peered":"2023-07-14T11:11:08.720636+0000","last_clean":"2023-07-14T11:11:08.720636+0000","last_became_active":"2023-07-14T11:09:42.600161+0000","last_became_peered":"2023-07-14T11:09:42.600161+0000","last_unstale":"2023-07-14T11:11:08.720636+0000","last_undegraded":"2023-07-14T11:11:08.720636+0000","last_fullsized":"2023-07-14T11:11:08.720636+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:28:23.282093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289308+0000","last_change":"2023-07-14T11:09:43.593633+0000","last_active":"2023-07-14T11:11:09.289308+0000","last_peered":"2023-07-14T11:11:09.289308+0000","last_clean":"2023-07-14T11:11:09.289308+0000","last_became_active":"2023-07-14T11:09:43.593257+0000","last_became_peered":"2023-07-14T11:09:43.593257+0000","last_unstale":"2023-07-14T11:11:09.289308+0000","last_undegraded":"2023-07-14T11:11:09.289308+0000","last_fullsized":"2023-07-14T11:11:09.289308+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:14:50.981268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,9],"acting":[0,1,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417664+0000","last_change":"2023-07-14T11:09:47.625425+0000","last_active":"2023-07-14T11:11:09.417664+0000","last_peered":"2023-07-14T11:11:09.417664+0000","last_clean":"2023-07-14T11:11:09.417664+0000","last_became_active":"2023-07-14T11:09:47.624739+0000","last_became_peered":"2023-07-14T11:09:47.624739+0000","last_unstale":"2023-07-14T11:11:09.417664+0000","last_undegraded":"2023-07-14T11:11:09.417664+0000","last_fullsized":"2023-07-14T11:11:09.417664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:09:37.313270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,9,5],"acting":[4,3,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1d","version":"90'12","reported_seq":43,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723372+0000","last_change":"2023-07-14T11:09:42.599404+0000","last_active":"2023-07-14T11:11:08.723372+0000","last_peered":"2023-07-14T11:11:08.723372+0000","last_clean":"2023-07-14T11:11:08.723372+0000","last_became_active":"2023-07-14T11:09:42.598926+0000","last_became_peered":"2023-07-14T11:09:42.598926+0000","last_unstale":"2023-07-14T11:11:08.723372+0000","last_undegraded":"2023-07-14T11:11:08.723372+0000","last_fullsized":"2023-07-14T11:11:08.723372+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":12,"log_dups_size":0,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:08:32.724455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":16,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":7,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001165+0000","last_change":"2023-07-14T11:09:43.591304+0000","last_active":"2023-07-14T11:11:09.001165+0000","last_peered":"2023-07-14T11:11:09.001165+0000","last_clean":"2023-07-14T11:11:09.001165+0000","last_became_active":"2023-07-14T11:09:43.590997+0000","last_became_peered":"2023-07-14T11:09:43.590997+0000","last_unstale":"2023-07-14T11:11:09.001165+0000","last_undegraded":"2023-07-14T11:11:09.001165+0000","last_fullsized":"2023-07-14T11:11:09.001165+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:32:43.730998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,6],"acting":[1,10,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417650+0000","last_change":"2023-07-14T11:09:47.619603+0000","last_active":"2023-07-14T11:11:09.417650+0000","last_peered":"2023-07-14T11:11:09.417650+0000","last_clean":"2023-07-14T11:11:09.417650+0000","last_became_active":"2023-07-14T11:09:47.619206+0000","last_became_peered":"2023-07-14T11:09:47.619206+0000","last_unstale":"2023-07-14T11:11:09.417650+0000","last_undegraded":"2023-07-14T11:11:09.417650+0000","last_fullsized":"2023-07-14T11:11:09.417650+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:01:46.106349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5,3],"acting":[4,6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1c","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723406+0000","last_change":"2023-07-14T11:09:42.594361+0000","last_active":"2023-07-14T11:11:08.723406+0000","last_peered":"2023-07-14T11:11:08.723406+0000","last_clean":"2023-07-14T11:11:08.723406+0000","last_became_active":"2023-07-14T11:09:42.593996+0000","last_became_peered":"2023-07-14T11:09:42.593996+0000","last_unstale":"2023-07-14T11:11:08.723406+0000","last_undegraded":"2023-07-14T11:11:08.723406+0000","last_fullsized":"2023-07-14T11:11:08.723406+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:16:53.790679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,1,5],"acting":[8,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719976+0000","last_change":"2023-07-14T11:09:43.593052+0000","last_active":"2023-07-14T11:11:08.719976+0000","last_peered":"2023-07-14T11:11:08.719976+0000","last_clean":"2023-07-14T11:11:08.719976+0000","last_became_active":"2023-07-14T11:09:43.592750+0000","last_became_peered":"2023-07-14T11:09:43.592750+0000","last_unstale":"2023-07-14T11:11:08.719976+0000","last_undegraded":"2023-07-14T11:11:08.719976+0000","last_fullsized":"2023-07-14T11:11:08.719976+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:57:18.583351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290083+0000","last_change":"2023-07-14T11:09:47.614472+0000","last_active":"2023-07-14T11:11:09.290083+0000","last_peered":"2023-07-14T11:11:09.290083+0000","last_clean":"2023-07-14T11:11:09.290083+0000","last_became_active":"2023-07-14T11:09:47.613315+0000","last_became_peered":"2023-07-14T11:09:47.613315+0000","last_unstale":"2023-07-14T11:11:09.290083+0000","last_undegraded":"2023-07-14T11:11:09.290083+0000","last_fullsized":"2023-07-14T11:11:09.290083+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:39:25.095450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7,11],"acting":[2,5,7,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1b","version":"90'5","reported_seq":135,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.509424+0000","last_change":"2023-07-14T11:09:42.598907+0000","last_active":"2023-07-14T11:11:50.509424+0000","last_peered":"2023-07-14T11:11:50.509424+0000","last_clean":"2023-07-14T11:11:50.509424+0000","last_became_active":"2023-07-14T11:09:42.598529+0000","last_became_peered":"2023-07-14T11:09:42.598529+0000","last_unstale":"2023-07-14T11:11:50.509424+0000","last_undegraded":"2023-07-14T11:11:50.509424+0000","last_fullsized":"2023-07-14T11:11:50.509424+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:51.482337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":574,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":100,"num_read_kb":2,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723544+0000","last_change":"2023-07-14T11:09:43.585350+0000","last_active":"2023-07-14T11:11:08.723544+0000","last_peered":"2023-07-14T11:11:08.723544+0000","last_clean":"2023-07-14T11:11:08.723544+0000","last_became_active":"2023-07-14T11:09:43.585053+0000","last_became_peered":"2023-07-14T11:09:43.585053+0000","last_unstale":"2023-07-14T11:11:08.723544+0000","last_undegraded":"2023-07-14T11:11:08.723544+0000","last_fullsized":"2023-07-14T11:11:08.723544+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:27:35.444809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,9,5],"acting":[8,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290389+0000","last_change":"2023-07-14T11:09:47.618070+0000","last_active":"2023-07-14T11:11:09.290389+0000","last_peered":"2023-07-14T11:11:09.290389+0000","last_clean":"2023-07-14T11:11:09.290389+0000","last_became_active":"2023-07-14T11:09:47.617677+0000","last_became_peered":"2023-07-14T11:09:47.617677+0000","last_unstale":"2023-07-14T11:11:09.290389+0000","last_undegraded":"2023-07-14T11:11:09.290389+0000","last_fullsized":"2023-07-14T11:11:09.290389+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:40.201892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,9,5],"acting":[2,6,9,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1a","version":"90'7","reported_seq":37,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722127+0000","last_change":"2023-07-14T11:09:42.597592+0000","last_active":"2023-07-14T11:11:08.722127+0000","last_peered":"2023-07-14T11:11:08.722127+0000","last_clean":"2023-07-14T11:11:08.722127+0000","last_became_active":"2023-07-14T11:09:42.597245+0000","last_became_peered":"2023-07-14T11:09:42.597245+0000","last_unstale":"2023-07-14T11:11:08.722127+0000","last_undegraded":"2023-07-14T11:11:08.722127+0000","last_fullsized":"2023-07-14T11:11:08.722127+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:59:36.711766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,6],"acting":[10,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719793+0000","last_change":"2023-07-14T11:10:07.745285+0000","last_active":"2023-07-14T11:11:08.719793+0000","last_peered":"2023-07-14T11:11:08.719793+0000","last_clean":"2023-07-14T11:11:08.719793+0000","last_became_active":"2023-07-14T11:10:07.744861+0000","last_became_peered":"2023-07-14T11:10:07.744861+0000","last_unstale":"2023-07-14T11:11:08.719793+0000","last_undegraded":"2023-07-14T11:11:08.719793+0000","last_fullsized":"2023-07-14T11:11:08.719793+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:20.276490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3],"acting":[5,8,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289569+0000","last_change":"2023-07-14T11:09:47.623750+0000","last_active":"2023-07-14T11:11:09.289569+0000","last_peered":"2023-07-14T11:11:09.289569+0000","last_clean":"2023-07-14T11:11:09.289569+0000","last_became_active":"2023-07-14T11:09:47.623352+0000","last_became_peered":"2023-07-14T11:09:47.623352+0000","last_unstale":"2023-07-14T11:11:09.289569+0000","last_undegraded":"2023-07-14T11:11:09.289569+0000","last_fullsized":"2023-07-14T11:11:09.289569+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:52:51.277511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,9,0],"acting":[11,4,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.19","version":"90'3","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718943+0000","last_change":"2023-07-14T11:09:42.586367+0000","last_active":"2023-07-14T11:11:08.718943+0000","last_peered":"2023-07-14T11:11:08.718943+0000","last_clean":"2023-07-14T11:11:08.718943+0000","last_became_active":"2023-07-14T11:09:42.586014+0000","last_became_peered":"2023-07-14T11:09:42.586014+0000","last_unstale":"2023-07-14T11:11:08.718943+0000","last_undegraded":"2023-07-14T11:11:08.718943+0000","last_fullsized":"2023-07-14T11:11:08.718943+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:11:40.222647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,9],"acting":[6,4,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719691+0000","last_change":"2023-07-14T11:09:43.583910+0000","last_active":"2023-07-14T11:11:08.719691+0000","last_peered":"2023-07-14T11:11:08.719691+0000","last_clean":"2023-07-14T11:11:08.719691+0000","last_became_active":"2023-07-14T11:09:43.583358+0000","last_became_peered":"2023-07-14T11:09:43.583358+0000","last_unstale":"2023-07-14T11:11:08.719691+0000","last_undegraded":"2023-07-14T11:11:08.719691+0000","last_fullsized":"2023-07-14T11:11:08.719691+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:36:10.169827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,6],"acting":[3,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289202+0000","last_change":"2023-07-14T11:09:47.622565+0000","last_active":"2023-07-14T11:11:09.289202+0000","last_peered":"2023-07-14T11:11:09.289202+0000","last_clean":"2023-07-14T11:11:09.289202+0000","last_became_active":"2023-07-14T11:09:47.622061+0000","last_became_peered":"2023-07-14T11:09:47.622061+0000","last_unstale":"2023-07-14T11:11:09.289202+0000","last_undegraded":"2023-07-14T11:11:09.289202+0000","last_fullsized":"2023-07-14T11:11:09.289202+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:08:58.024785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6,1],"acting":[0,7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.18","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719453+0000","last_change":"2023-07-14T11:09:42.601003+0000","last_active":"2023-07-14T11:11:08.719453+0000","last_peered":"2023-07-14T11:11:08.719453+0000","last_clean":"2023-07-14T11:11:08.719453+0000","last_became_active":"2023-07-14T11:09:42.600701+0000","last_became_peered":"2023-07-14T11:09:42.600701+0000","last_unstale":"2023-07-14T11:11:08.719453+0000","last_undegraded":"2023-07-14T11:11:08.719453+0000","last_fullsized":"2023-07-14T11:11:08.719453+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.383791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289719+0000","last_change":"2023-07-14T11:09:43.599647+0000","last_active":"2023-07-14T11:11:09.289719+0000","last_peered":"2023-07-14T11:11:09.289719+0000","last_clean":"2023-07-14T11:11:09.289719+0000","last_became_active":"2023-07-14T11:09:43.599293+0000","last_became_peered":"2023-07-14T11:09:43.599293+0000","last_unstale":"2023-07-14T11:11:09.289719+0000","last_undegraded":"2023-07-14T11:11:09.289719+0000","last_fullsized":"2023-07-14T11:11:09.289719+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:36:52.817530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,10],"acting":[9,3,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719974+0000","last_change":"2023-07-14T11:09:47.616642+0000","last_active":"2023-07-14T11:11:08.719974+0000","last_peered":"2023-07-14T11:11:08.719974+0000","last_clean":"2023-07-14T11:11:08.719974+0000","last_became_active":"2023-07-14T11:09:47.616163+0000","last_became_peered":"2023-07-14T11:09:47.616163+0000","last_unstale":"2023-07-14T11:11:08.719974+0000","last_undegraded":"2023-07-14T11:11:08.719974+0000","last_fullsized":"2023-07-14T11:11:08.719974+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:20:52.884869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1,5],"acting":[3,7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.17","version":"90'4","reported_seq":232,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:53.569582+0000","last_change":"2023-07-14T11:09:42.598209+0000","last_active":"2023-07-14T11:11:53.569582+0000","last_peered":"2023-07-14T11:11:53.569582+0000","last_clean":"2023-07-14T11:11:53.569582+0000","last_became_active":"2023-07-14T11:09:42.597839+0000","last_became_peered":"2023-07-14T11:09:42.597839+0000","last_unstale":"2023-07-14T11:11:53.569582+0000","last_undegraded":"2023-07-14T11:11:53.569582+0000","last_fullsized":"2023-07-14T11:11:53.569582+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:44:32.109970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":198,"num_read_kb":198,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,5,9],"acting":[10,5,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001506+0000","last_change":"2023-07-14T11:09:43.591723+0000","last_active":"2023-07-14T11:11:09.001506+0000","last_peered":"2023-07-14T11:11:09.001506+0000","last_clean":"2023-07-14T11:11:09.001506+0000","last_became_active":"2023-07-14T11:09:43.591440+0000","last_became_peered":"2023-07-14T11:09:43.591440+0000","last_unstale":"2023-07-14T11:11:09.001506+0000","last_undegraded":"2023-07-14T11:11:09.001506+0000","last_fullsized":"2023-07-14T11:11:09.001506+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:09:15.291405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,10,9],"acting":[1,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289640+0000","last_change":"2023-07-14T11:09:47.623536+0000","last_active":"2023-07-14T11:11:09.289640+0000","last_peered":"2023-07-14T11:11:09.289640+0000","last_clean":"2023-07-14T11:11:09.289640+0000","last_became_active":"2023-07-14T11:09:47.623099+0000","last_became_peered":"2023-07-14T11:09:47.623099+0000","last_unstale":"2023-07-14T11:11:09.289640+0000","last_undegraded":"2023-07-14T11:11:09.289640+0000","last_fullsized":"2023-07-14T11:11:09.289640+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:45.979895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,0,2,4],"acting":[9,0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.16","version":"90'4","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289621+0000","last_change":"2023-07-14T11:09:42.598806+0000","last_active":"2023-07-14T11:11:09.289621+0000","last_peered":"2023-07-14T11:11:09.289621+0000","last_clean":"2023-07-14T11:11:09.289621+0000","last_became_active":"2023-07-14T11:09:42.598431+0000","last_became_peered":"2023-07-14T11:09:42.598431+0000","last_unstale":"2023-07-14T11:11:09.289621+0000","last_undegraded":"2023-07-14T11:11:09.289621+0000","last_fullsized":"2023-07-14T11:11:09.289621+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:25:56.170940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289258+0000","last_change":"2023-07-14T11:09:43.589632+0000","last_active":"2023-07-14T11:11:09.289258+0000","last_peered":"2023-07-14T11:11:09.289258+0000","last_clean":"2023-07-14T11:11:09.289258+0000","last_became_active":"2023-07-14T11:09:43.589207+0000","last_became_peered":"2023-07-14T11:09:43.589207+0000","last_unstale":"2023-07-14T11:11:09.289258+0000","last_undegraded":"2023-07-14T11:11:09.289258+0000","last_fullsized":"2023-07-14T11:11:09.289258+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:08.488544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417841+0000","last_change":"2023-07-14T11:09:47.625116+0000","last_active":"2023-07-14T11:11:09.417841+0000","last_peered":"2023-07-14T11:11:09.417841+0000","last_clean":"2023-07-14T11:11:09.417841+0000","last_became_active":"2023-07-14T11:09:47.624134+0000","last_became_peered":"2023-07-14T11:09:47.624134+0000","last_unstale":"2023-07-14T11:11:09.417841+0000","last_undegraded":"2023-07-14T11:11:09.417841+0000","last_fullsized":"2023-07-14T11:11:09.417841+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:28:53.190914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2,9],"acting":[4,1,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.15","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719886+0000","last_change":"2023-07-14T11:09:42.604332+0000","last_active":"2023-07-14T11:11:08.719886+0000","last_peered":"2023-07-14T11:11:08.719886+0000","last_clean":"2023-07-14T11:11:08.719886+0000","last_became_active":"2023-07-14T11:09:42.603936+0000","last_became_peered":"2023-07-14T11:09:42.603936+0000","last_unstale":"2023-07-14T11:11:08.719886+0000","last_undegraded":"2023-07-14T11:11:08.719886+0000","last_fullsized":"2023-07-14T11:11:08.719886+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:21:22.302853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":672,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,10],"acting":[5,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001306+0000","last_change":"2023-07-14T11:10:07.738962+0000","last_active":"2023-07-14T11:11:09.001306+0000","last_peered":"2023-07-14T11:11:09.001306+0000","last_clean":"2023-07-14T11:11:09.001306+0000","last_became_active":"2023-07-14T11:10:07.738571+0000","last_became_peered":"2023-07-14T11:10:07.738571+0000","last_unstale":"2023-07-14T11:11:09.001306+0000","last_undegraded":"2023-07-14T11:11:09.001306+0000","last_fullsized":"2023-07-14T11:11:09.001306+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:55:41.153671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,8,6],"acting":[1,8,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.293052+0000","last_change":"2023-07-14T11:11:09.292716+0000","last_active":"2023-07-14T11:11:09.293052+0000","last_peered":"2023-07-14T11:11:09.293052+0000","last_clean":"2023-07-14T11:11:09.293052+0000","last_became_active":"2023-07-14T11:11:09.292262+0000","last_became_peered":"2023-07-14T11:11:09.292262+0000","last_unstale":"2023-07-14T11:11:09.293052+0000","last_undegraded":"2023-07-14T11:11:09.293052+0000","last_fullsized":"2023-07-14T11:11:09.293052+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:44:07.811874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,8,11,2],"acting":[10,8,11,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"2.14","version":"92'16","reported_seq":249,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:53.548802+0000","last_change":"2023-07-14T11:09:42.592035+0000","last_active":"2023-07-14T11:11:53.548802+0000","last_peered":"2023-07-14T11:11:53.548802+0000","last_clean":"2023-07-14T11:11:53.548802+0000","last_became_active":"2023-07-14T11:09:42.591720+0000","last_became_peered":"2023-07-14T11:09:42.591720+0000","last_unstale":"2023-07-14T11:11:53.548802+0000","last_undegraded":"2023-07-14T11:11:53.548802+0000","last_fullsized":"2023-07-14T11:11:53.548802+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":16,"log_dups_size":0,"ondisk_log_size":16,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:44:43.522939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4052,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":203,"num_read_kb":106,"num_write":18,"num_write_kb":10,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,1,2],"acting":[10,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723095+0000","last_change":"2023-07-14T11:09:43.590581+0000","last_active":"2023-07-14T11:11:08.723095+0000","last_peered":"2023-07-14T11:11:08.723095+0000","last_clean":"2023-07-14T11:11:08.723095+0000","last_became_active":"2023-07-14T11:09:43.590291+0000","last_became_peered":"2023-07-14T11:09:43.590291+0000","last_unstale":"2023-07-14T11:11:08.723095+0000","last_undegraded":"2023-07-14T11:11:08.723095+0000","last_fullsized":"2023-07-14T11:11:08.723095+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:51:23.455018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,7],"acting":[8,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720210+0000","last_change":"2023-07-14T11:09:47.618423+0000","last_active":"2023-07-14T11:11:08.720210+0000","last_peered":"2023-07-14T11:11:08.720210+0000","last_clean":"2023-07-14T11:11:08.720210+0000","last_became_active":"2023-07-14T11:09:47.618035+0000","last_became_peered":"2023-07-14T11:09:47.618035+0000","last_unstale":"2023-07-14T11:11:08.720210+0000","last_undegraded":"2023-07-14T11:11:08.720210+0000","last_fullsized":"2023-07-14T11:11:08.720210+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:35:42.833169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,8,3,6],"acting":[5,8,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.13","version":"90'9","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290243+0000","last_change":"2023-07-14T11:09:42.597600+0000","last_active":"2023-07-14T11:11:09.290243+0000","last_peered":"2023-07-14T11:11:09.290243+0000","last_clean":"2023-07-14T11:11:09.290243+0000","last_became_active":"2023-07-14T11:09:42.597251+0000","last_became_peered":"2023-07-14T11:09:42.597251+0000","last_unstale":"2023-07-14T11:11:09.290243+0000","last_undegraded":"2023-07-14T11:11:09.290243+0000","last_fullsized":"2023-07-14T11:11:09.290243+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:34.169535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":13,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,10,3],"acting":[2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719060+0000","last_change":"2023-07-14T11:09:43.592157+0000","last_active":"2023-07-14T11:11:08.719060+0000","last_peered":"2023-07-14T11:11:08.719060+0000","last_clean":"2023-07-14T11:11:08.719060+0000","last_became_active":"2023-07-14T11:09:43.591574+0000","last_became_peered":"2023-07-14T11:09:43.591574+0000","last_unstale":"2023-07-14T11:11:08.719060+0000","last_undegraded":"2023-07-14T11:11:08.719060+0000","last_fullsized":"2023-07-14T11:11:08.719060+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:16:24.691976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290657+0000","last_change":"2023-07-14T11:10:07.744522+0000","last_active":"2023-07-14T11:11:09.290657+0000","last_peered":"2023-07-14T11:11:09.290657+0000","last_clean":"2023-07-14T11:11:09.290657+0000","last_became_active":"2023-07-14T11:10:07.744032+0000","last_became_peered":"2023-07-14T11:10:07.744032+0000","last_unstale":"2023-07-14T11:11:09.290657+0000","last_undegraded":"2023-07-14T11:11:09.290657+0000","last_fullsized":"2023-07-14T11:11:09.290657+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:53:37.211874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,7,6],"acting":[11,8,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.12","version":"90'5","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720108+0000","last_change":"2023-07-14T11:09:42.599158+0000","last_active":"2023-07-14T11:11:08.720108+0000","last_peered":"2023-07-14T11:11:08.720108+0000","last_clean":"2023-07-14T11:11:08.720108+0000","last_became_active":"2023-07-14T11:09:42.598811+0000","last_became_peered":"2023-07-14T11:09:42.598811+0000","last_unstale":"2023-07-14T11:11:08.720108+0000","last_undegraded":"2023-07-14T11:11:08.720108+0000","last_fullsized":"2023-07-14T11:11:08.720108+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:55:08.685860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289465+0000","last_change":"2023-07-14T11:09:43.592897+0000","last_active":"2023-07-14T11:11:09.289465+0000","last_peered":"2023-07-14T11:11:09.289465+0000","last_clean":"2023-07-14T11:11:09.289465+0000","last_became_active":"2023-07-14T11:09:43.592145+0000","last_became_peered":"2023-07-14T11:09:43.592145+0000","last_unstale":"2023-07-14T11:11:09.289465+0000","last_undegraded":"2023-07-14T11:11:09.289465+0000","last_fullsized":"2023-07-14T11:11:09.289465+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:07:02.366836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290356+0000","last_change":"2023-07-14T11:09:47.624611+0000","last_active":"2023-07-14T11:11:09.290356+0000","last_peered":"2023-07-14T11:11:09.290356+0000","last_clean":"2023-07-14T11:11:09.290356+0000","last_became_active":"2023-07-14T11:09:47.623511+0000","last_became_peered":"2023-07-14T11:09:47.623511+0000","last_unstale":"2023-07-14T11:11:09.290356+0000","last_undegraded":"2023-07-14T11:11:09.290356+0000","last_fullsized":"2023-07-14T11:11:09.290356+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:04:23.808018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,7,0,3],"acting":[9,7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.11","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.521863+0000","last_change":"2023-07-14T11:09:42.599463+0000","last_active":"2023-07-14T11:11:50.521863+0000","last_peered":"2023-07-14T11:11:50.521863+0000","last_clean":"2023-07-14T11:11:50.521863+0000","last_became_active":"2023-07-14T11:09:42.598950+0000","last_became_peered":"2023-07-14T11:09:42.598950+0000","last_unstale":"2023-07-14T11:11:50.521863+0000","last_undegraded":"2023-07-14T11:11:50.521863+0000","last_fullsized":"2023-07-14T11:11:50.521863+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:08.806584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":3,"num_write":5,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,10],"acting":[4,7,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289179+0000","last_change":"2023-07-14T11:09:43.589371+0000","last_active":"2023-07-14T11:11:09.289179+0000","last_peered":"2023-07-14T11:11:09.289179+0000","last_clean":"2023-07-14T11:11:09.289179+0000","last_became_active":"2023-07-14T11:09:43.589081+0000","last_became_peered":"2023-07-14T11:09:43.589081+0000","last_unstale":"2023-07-14T11:11:09.289179+0000","last_undegraded":"2023-07-14T11:11:09.289179+0000","last_fullsized":"2023-07-14T11:11:09.289179+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:00.991168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719386+0000","last_change":"2023-07-14T11:09:47.628668+0000","last_active":"2023-07-14T11:11:08.719386+0000","last_peered":"2023-07-14T11:11:08.719386+0000","last_clean":"2023-07-14T11:11:08.719386+0000","last_became_active":"2023-07-14T11:09:47.628210+0000","last_became_peered":"2023-07-14T11:09:47.628210+0000","last_unstale":"2023-07-14T11:11:08.719386+0000","last_undegraded":"2023-07-14T11:11:08.719386+0000","last_fullsized":"2023-07-14T11:11:08.719386+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:10:03.730236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,9,11],"acting":[6,3,9,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"90'17","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722670+0000","last_change":"2023-07-14T11:09:42.599511+0000","last_active":"2023-07-14T11:11:08.722670+0000","last_peered":"2023-07-14T11:11:08.722670+0000","last_clean":"2023-07-14T11:11:08.722670+0000","last_became_active":"2023-07-14T11:09:42.599154+0000","last_became_peered":"2023-07-14T11:09:42.599154+0000","last_unstale":"2023-07-14T11:11:08.722670+0000","last_undegraded":"2023-07-14T11:11:08.722670+0000","last_fullsized":"2023-07-14T11:11:08.722670+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:46:28.743826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":20,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":6,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,2],"acting":[8,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719745+0000","last_change":"2023-07-14T11:09:43.586933+0000","last_active":"2023-07-14T11:11:08.719745+0000","last_peered":"2023-07-14T11:11:08.719745+0000","last_clean":"2023-07-14T11:11:08.719745+0000","last_became_active":"2023-07-14T11:09:43.586639+0000","last_became_peered":"2023-07-14T11:09:43.586639+0000","last_unstale":"2023-07-14T11:11:08.719745+0000","last_undegraded":"2023-07-14T11:11:08.719745+0000","last_fullsized":"2023-07-14T11:11:08.719745+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:26:00.226723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719524+0000","last_change":"2023-07-14T11:09:47.613254+0000","last_active":"2023-07-14T11:11:08.719524+0000","last_peered":"2023-07-14T11:11:08.719524+0000","last_clean":"2023-07-14T11:11:08.719524+0000","last_became_active":"2023-07-14T11:09:47.612784+0000","last_became_peered":"2023-07-14T11:09:47.612784+0000","last_unstale":"2023-07-14T11:11:08.719524+0000","last_undegraded":"2023-07-14T11:11:08.719524+0000","last_fullsized":"2023-07-14T11:11:08.719524+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:07:30.579895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3,1],"acting":[5,0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.f","version":"90'2","reported_seq":33,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723335+0000","last_change":"2023-07-14T11:09:42.590981+0000","last_active":"2023-07-14T11:11:08.723335+0000","last_peered":"2023-07-14T11:11:08.723335+0000","last_clean":"2023-07-14T11:11:08.723335+0000","last_became_active":"2023-07-14T11:09:42.590581+0000","last_became_peered":"2023-07-14T11:09:42.590581+0000","last_unstale":"2023-07-14T11:11:08.723335+0000","last_undegraded":"2023-07-14T11:11:08.723335+0000","last_fullsized":"2023-07-14T11:11:08.723335+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:05:23.398441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":2,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,5,4],"acting":[8,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291267+0000","last_change":"2023-07-14T11:09:43.585650+0000","last_active":"2023-07-14T11:11:09.291267+0000","last_peered":"2023-07-14T11:11:09.291267+0000","last_clean":"2023-07-14T11:11:09.291267+0000","last_became_active":"2023-07-14T11:09:43.585318+0000","last_became_peered":"2023-07-14T11:09:43.585318+0000","last_unstale":"2023-07-14T11:11:09.291267+0000","last_undegraded":"2023-07-14T11:11:09.291267+0000","last_fullsized":"2023-07-14T11:11:09.291267+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:13:20.237238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,4],"acting":[11,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291884+0000","last_change":"2023-07-14T11:11:09.291884+0000","last_active":"2023-07-14T11:11:09.291884+0000","last_peered":"2023-07-14T11:11:09.291884+0000","last_clean":"2023-07-14T11:11:09.291884+0000","last_became_active":"2023-07-14T11:11:09.291383+0000","last_became_peered":"2023-07-14T11:11:09.291383+0000","last_unstale":"2023-07-14T11:11:09.291884+0000","last_undegraded":"2023-07-14T11:11:09.291884+0000","last_fullsized":"2023-07-14T11:11:09.291884+0000","mapping_epoch":91,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":92,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:58:27.329172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,10,11,1],"acting":[8,10,11,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"2.4","version":"90'1","reported_seq":32,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290030+0000","last_change":"2023-07-14T11:09:42.598212+0000","last_active":"2023-07-14T11:11:09.290030+0000","last_peered":"2023-07-14T11:11:09.290030+0000","last_clean":"2023-07-14T11:11:09.290030+0000","last_became_active":"2023-07-14T11:09:42.597854+0000","last_became_peered":"2023-07-14T11:09:42.597854+0000","last_unstale":"2023-07-14T11:11:09.290030+0000","last_undegraded":"2023-07-14T11:11:09.290030+0000","last_fullsized":"2023-07-14T11:11:09.290030+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:49:08.911423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,2,0],"acting":[9,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721739+0000","last_change":"2023-07-14T11:09:43.596155+0000","last_active":"2023-07-14T11:11:08.721739+0000","last_peered":"2023-07-14T11:11:08.721739+0000","last_clean":"2023-07-14T11:11:08.721739+0000","last_became_active":"2023-07-14T11:09:43.595848+0000","last_became_peered":"2023-07-14T11:09:43.595848+0000","last_unstale":"2023-07-14T11:11:08.721739+0000","last_undegraded":"2023-07-14T11:11:08.721739+0000","last_fullsized":"2023-07-14T11:11:08.721739+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:02.329622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,3,6],"acting":[10,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719474+0000","last_change":"2023-07-14T11:09:47.618315+0000","last_active":"2023-07-14T11:11:08.719474+0000","last_peered":"2023-07-14T11:11:08.719474+0000","last_clean":"2023-07-14T11:11:08.719474+0000","last_became_active":"2023-07-14T11:09:47.617926+0000","last_became_peered":"2023-07-14T11:09:47.617926+0000","last_unstale":"2023-07-14T11:11:08.719474+0000","last_undegraded":"2023-07-14T11:11:08.719474+0000","last_fullsized":"2023-07-14T11:11:08.719474+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:54:56.816899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1,8],"acting":[6,0,1,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"90'5","reported_seq":130,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:49.552173+0000","last_change":"2023-07-14T11:09:42.602757+0000","last_active":"2023-07-14T11:11:49.552173+0000","last_peered":"2023-07-14T11:11:49.552173+0000","last_clean":"2023-07-14T11:11:49.552173+0000","last_became_active":"2023-07-14T11:09:42.602368+0000","last_became_peered":"2023-07-14T11:09:42.602368+0000","last_unstale":"2023-07-14T11:11:49.552173+0000","last_undegraded":"2023-07-14T11:11:49.552173+0000","last_fullsized":"2023-07-14T11:11:49.552173+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:44:42.890747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":540,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":94,"num_read_kb":1,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.721512+0000","last_change":"2023-07-14T11:09:43.593054+0000","last_active":"2023-07-14T11:11:08.721512+0000","last_peered":"2023-07-14T11:11:08.721512+0000","last_clean":"2023-07-14T11:11:08.721512+0000","last_became_active":"2023-07-14T11:09:43.592717+0000","last_became_peered":"2023-07-14T11:09:43.592717+0000","last_unstale":"2023-07-14T11:11:08.721512+0000","last_undegraded":"2023-07-14T11:11:08.721512+0000","last_fullsized":"2023-07-14T11:11:08.721512+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:00:06.643656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[10,4,0],"acting":[10,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":10,"acting_primary":10,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719363+0000","last_change":"2023-07-14T11:09:47.737094+0000","last_active":"2023-07-14T11:11:08.719363+0000","last_peered":"2023-07-14T11:11:08.719363+0000","last_clean":"2023-07-14T11:11:08.719363+0000","last_became_active":"2023-07-14T11:09:47.736548+0000","last_became_peered":"2023-07-14T11:09:47.736548+0000","last_unstale":"2023-07-14T11:11:08.719363+0000","last_undegraded":"2023-07-14T11:11:08.719363+0000","last_fullsized":"2023-07-14T11:11:08.719363+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:59:01.913296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,10,0],"acting":[6,5,10,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719845+0000","last_change":"2023-07-14T11:09:43.590525+0000","last_active":"2023-07-14T11:11:08.719845+0000","last_peered":"2023-07-14T11:11:08.719845+0000","last_clean":"2023-07-14T11:11:08.719845+0000","last_became_active":"2023-07-14T11:09:43.590207+0000","last_became_peered":"2023-07-14T11:09:43.590207+0000","last_unstale":"2023-07-14T11:11:08.719845+0000","last_undegraded":"2023-07-14T11:11:08.719845+0000","last_fullsized":"2023-07-14T11:11:08.719845+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:21:32.430525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"90'5","reported_seq":134,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:52.523992+0000","last_change":"2023-07-14T11:09:42.595837+0000","last_active":"2023-07-14T11:11:52.523992+0000","last_peered":"2023-07-14T11:11:52.523992+0000","last_clean":"2023-07-14T11:11:52.523992+0000","last_became_active":"2023-07-14T11:09:42.595480+0000","last_became_peered":"2023-07-14T11:09:42.595480+0000","last_unstale":"2023-07-14T11:11:52.523992+0000","last_undegraded":"2023-07-14T11:11:52.523992+0000","last_fullsized":"2023-07-14T11:11:52.523992+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:37:13.796130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":98,"num_read_kb":98,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289479+0000","last_change":"2023-07-14T11:09:47.624642+0000","last_active":"2023-07-14T11:11:09.289479+0000","last_peered":"2023-07-14T11:11:09.289479+0000","last_clean":"2023-07-14T11:11:09.289479+0000","last_became_active":"2023-07-14T11:09:47.624200+0000","last_became_peered":"2023-07-14T11:09:47.624200+0000","last_unstale":"2023-07-14T11:11:09.289479+0000","last_undegraded":"2023-07-14T11:11:09.289479+0000","last_fullsized":"2023-07-14T11:11:09.289479+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:41:35.094180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,11,9],"acting":[7,5,11,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719797+0000","last_change":"2023-07-14T11:09:43.590978+0000","last_active":"2023-07-14T11:11:08.719797+0000","last_peered":"2023-07-14T11:11:08.719797+0000","last_clean":"2023-07-14T11:11:08.719797+0000","last_became_active":"2023-07-14T11:09:43.590663+0000","last_became_peered":"2023-07-14T11:09:43.590663+0000","last_unstale":"2023-07-14T11:11:08.719797+0000","last_undegraded":"2023-07-14T11:11:08.719797+0000","last_fullsized":"2023-07-14T11:11:08.719797+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:19:31.917453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"90'6","reported_seq":40,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417870+0000","last_change":"2023-07-14T11:09:42.590241+0000","last_active":"2023-07-14T11:11:09.417870+0000","last_peered":"2023-07-14T11:11:09.417870+0000","last_clean":"2023-07-14T11:11:09.417870+0000","last_became_active":"2023-07-14T11:09:42.589869+0000","last_became_peered":"2023-07-14T11:09:42.589869+0000","last_unstale":"2023-07-14T11:11:09.417870+0000","last_undegraded":"2023-07-14T11:11:09.417870+0000","last_fullsized":"2023-07-14T11:11:09.417870+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:36:12.795737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":8,"num_write_kb":8,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,10,9],"acting":[4,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"22'137","reported_seq":359,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289040+0000","last_change":"2023-07-14T11:08:37.839514+0000","last_active":"2023-07-14T11:11:09.289040+0000","last_peered":"2023-07-14T11:11:09.289040+0000","last_clean":"2023-07-14T11:11:09.289040+0000","last_became_active":"2023-07-14T11:08:37.839127+0000","last_became_peered":"2023-07-14T11:08:37.839127+0000","last_unstale":"2023-07-14T11:11:09.289040+0000","last_undegraded":"2023-07-14T11:11:09.289040+0000","last_fullsized":"2023-07-14T11:11:09.289040+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:05:27.186868+0000","last_clean_scrub_stamp":"2023-07-14T11:05:27.186868+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:28:25.142388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,10],"acting":[7,5,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720162+0000","last_change":"2023-07-14T11:09:47.737004+0000","last_active":"2023-07-14T11:11:08.720162+0000","last_peered":"2023-07-14T11:11:08.720162+0000","last_clean":"2023-07-14T11:11:08.720162+0000","last_became_active":"2023-07-14T11:09:47.736554+0000","last_became_peered":"2023-07-14T11:09:47.736554+0000","last_unstale":"2023-07-14T11:11:08.720162+0000","last_undegraded":"2023-07-14T11:11:08.720162+0000","last_fullsized":"2023-07-14T11:11:08.720162+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:39:21.892266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,6,10],"acting":[5,9,6,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.718829+0000","last_change":"2023-07-14T11:09:43.589810+0000","last_active":"2023-07-14T11:11:08.718829+0000","last_peered":"2023-07-14T11:11:08.718829+0000","last_clean":"2023-07-14T11:11:08.718829+0000","last_became_active":"2023-07-14T11:09:43.589488+0000","last_became_peered":"2023-07-14T11:09:43.589488+0000","last_unstale":"2023-07-14T11:11:08.718829+0000","last_undegraded":"2023-07-14T11:11:08.718829+0000","last_fullsized":"2023-07-14T11:11:08.718829+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:43:18.820300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"90'4","reported_seq":41,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289160+0000","last_change":"2023-07-14T11:10:07.742886+0000","last_active":"2023-07-14T11:11:09.289160+0000","last_peered":"2023-07-14T11:11:09.289160+0000","last_clean":"2023-07-14T11:11:09.289160+0000","last_became_active":"2023-07-14T11:10:07.742479+0000","last_became_peered":"2023-07-14T11:10:07.742479+0000","last_unstale":"2023-07-14T11:11:09.289160+0000","last_undegraded":"2023-07-14T11:11:09.289160+0000","last_fullsized":"2023-07-14T11:11:09.289160+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:45:59.013859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289968+0000","last_change":"2023-07-14T11:09:47.619888+0000","last_active":"2023-07-14T11:11:09.289968+0000","last_peered":"2023-07-14T11:11:09.289968+0000","last_clean":"2023-07-14T11:11:09.289968+0000","last_became_active":"2023-07-14T11:09:47.619453+0000","last_became_peered":"2023-07-14T11:09:47.619453+0000","last_unstale":"2023-07-14T11:11:09.289968+0000","last_undegraded":"2023-07-14T11:11:09.289968+0000","last_fullsized":"2023-07-14T11:11:09.289968+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:29:24.636531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,5,7,6],"acting":[11,5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"2.5","version":"90'4","reported_seq":36,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.723306+0000","last_change":"2023-07-14T11:09:42.604297+0000","last_active":"2023-07-14T11:11:08.723306+0000","last_peered":"2023-07-14T11:11:08.723306+0000","last_clean":"2023-07-14T11:11:08.723306+0000","last_became_active":"2023-07-14T11:09:42.603937+0000","last_became_peered":"2023-07-14T11:09:42.603937+0000","last_unstale":"2023-07-14T11:11:08.723306+0000","last_undegraded":"2023-07-14T11:11:08.723306+0000","last_fullsized":"2023-07-14T11:11:08.723306+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:26:37.536988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,0,9],"acting":[8,0,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719233+0000","last_change":"2023-07-14T11:09:43.589934+0000","last_active":"2023-07-14T11:11:08.719233+0000","last_peered":"2023-07-14T11:11:08.719233+0000","last_clean":"2023-07-14T11:11:08.719233+0000","last_became_active":"2023-07-14T11:09:43.589600+0000","last_became_peered":"2023-07-14T11:09:43.589600+0000","last_unstale":"2023-07-14T11:11:08.719233+0000","last_undegraded":"2023-07-14T11:11:08.719233+0000","last_fullsized":"2023-07-14T11:11:08.719233+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:24:03.090653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,9],"acting":[6,2,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719888+0000","last_change":"2023-07-14T11:09:47.736972+0000","last_active":"2023-07-14T11:11:08.719888+0000","last_peered":"2023-07-14T11:11:08.719888+0000","last_clean":"2023-07-14T11:11:08.719888+0000","last_became_active":"2023-07-14T11:09:47.736549+0000","last_became_peered":"2023-07-14T11:09:47.736549+0000","last_unstale":"2023-07-14T11:11:08.719888+0000","last_undegraded":"2023-07-14T11:11:08.719888+0000","last_fullsized":"2023-07-14T11:11:08.719888+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:40:53.362733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,10,0,7],"acting":[3,10,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"90'7","reported_seq":48,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290126+0000","last_change":"2023-07-14T11:10:07.748353+0000","last_active":"2023-07-14T11:11:09.290126+0000","last_peered":"2023-07-14T11:11:09.290126+0000","last_clean":"2023-07-14T11:11:09.290126+0000","last_became_active":"2023-07-14T11:10:07.744143+0000","last_became_peered":"2023-07-14T11:10:07.744143+0000","last_unstale":"2023-07-14T11:11:09.290126+0000","last_undegraded":"2023-07-14T11:11:09.290126+0000","last_fullsized":"2023-07-14T11:11:09.290126+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:27:46.858212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,11,3],"acting":[9,11,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720504+0000","last_change":"2023-07-14T11:09:43.588256+0000","last_active":"2023-07-14T11:11:08.720504+0000","last_peered":"2023-07-14T11:11:08.720504+0000","last_clean":"2023-07-14T11:11:08.720504+0000","last_became_active":"2023-07-14T11:09:43.587869+0000","last_became_peered":"2023-07-14T11:09:43.587869+0000","last_unstale":"2023-07-14T11:11:08.720504+0000","last_undegraded":"2023-07-14T11:11:08.720504+0000","last_fullsized":"2023-07-14T11:11:08.720504+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:36:05.284617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,11,7],"acting":[3,11,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290123+0000","last_change":"2023-07-14T11:09:47.616924+0000","last_active":"2023-07-14T11:11:09.290123+0000","last_peered":"2023-07-14T11:11:09.290123+0000","last_clean":"2023-07-14T11:11:09.290123+0000","last_became_active":"2023-07-14T11:09:47.616521+0000","last_became_peered":"2023-07-14T11:09:47.616521+0000","last_unstale":"2023-07-14T11:11:09.290123+0000","last_undegraded":"2023-07-14T11:11:09.290123+0000","last_fullsized":"2023-07-14T11:11:09.290123+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:02:54.462119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4,7],"acting":[2,1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"90'7","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418279+0000","last_change":"2023-07-14T11:09:42.594329+0000","last_active":"2023-07-14T11:11:09.418279+0000","last_peered":"2023-07-14T11:11:09.418279+0000","last_clean":"2023-07-14T11:11:09.418279+0000","last_became_active":"2023-07-14T11:09:42.593726+0000","last_became_peered":"2023-07-14T11:09:42.593726+0000","last_unstale":"2023-07-14T11:11:09.418279+0000","last_undegraded":"2023-07-14T11:11:09.418279+0000","last_fullsized":"2023-07-14T11:11:09.418279+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:28:55.029604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,11,6],"acting":[4,11,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289355+0000","last_change":"2023-07-14T11:09:43.592047+0000","last_active":"2023-07-14T11:11:09.289355+0000","last_peered":"2023-07-14T11:11:09.289355+0000","last_clean":"2023-07-14T11:11:09.289355+0000","last_became_active":"2023-07-14T11:09:43.591688+0000","last_became_peered":"2023-07-14T11:09:43.591688+0000","last_unstale":"2023-07-14T11:11:09.289355+0000","last_undegraded":"2023-07-14T11:11:09.289355+0000","last_fullsized":"2023-07-14T11:11:09.289355+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:35:10.693230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.418030+0000","last_change":"2023-07-14T11:09:47.736881+0000","last_active":"2023-07-14T11:11:09.418030+0000","last_peered":"2023-07-14T11:11:09.418030+0000","last_clean":"2023-07-14T11:11:09.418030+0000","last_became_active":"2023-07-14T11:09:47.736440+0000","last_became_peered":"2023-07-14T11:09:47.736440+0000","last_unstale":"2023-07-14T11:11:09.418030+0000","last_undegraded":"2023-07-14T11:11:09.418030+0000","last_fullsized":"2023-07-14T11:11:09.418030+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:44:16.733738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,10,9],"acting":[4,5,10,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.8","version":"90'2","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000828+0000","last_change":"2023-07-14T11:10:08.143283+0000","last_active":"2023-07-14T11:11:09.000828+0000","last_peered":"2023-07-14T11:11:09.000828+0000","last_clean":"2023-07-14T11:11:09.000828+0000","last_became_active":"2023-07-14T11:10:08.142868+0000","last_became_peered":"2023-07-14T11:10:08.142868+0000","last_unstale":"2023-07-14T11:11:09.000828+0000","last_undegraded":"2023-07-14T11:11:09.000828+0000","last_fullsized":"2023-07-14T11:11:09.000828+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:33:19.700027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,11,0],"acting":[1,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722662+0000","last_change":"2023-07-14T11:09:43.586157+0000","last_active":"2023-07-14T11:11:08.722662+0000","last_peered":"2023-07-14T11:11:08.722662+0000","last_clean":"2023-07-14T11:11:08.722662+0000","last_became_active":"2023-07-14T11:09:43.585839+0000","last_became_peered":"2023-07-14T11:09:43.585839+0000","last_unstale":"2023-07-14T11:11:08.722662+0000","last_undegraded":"2023-07-14T11:11:08.722662+0000","last_fullsized":"2023-07-14T11:11:08.722662+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:14:20.212946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,11,10],"acting":[8,11,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290230+0000","last_change":"2023-07-14T11:09:47.617715+0000","last_active":"2023-07-14T11:11:09.290230+0000","last_peered":"2023-07-14T11:11:09.290230+0000","last_clean":"2023-07-14T11:11:09.290230+0000","last_became_active":"2023-07-14T11:09:47.616018+0000","last_became_peered":"2023-07-14T11:09:47.616018+0000","last_unstale":"2023-07-14T11:11:09.290230+0000","last_undegraded":"2023-07-14T11:11:09.290230+0000","last_fullsized":"2023-07-14T11:11:09.290230+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T16:02:32.312663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,3,2,5],"acting":[9,3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"2.9","version":"90'7","reported_seq":38,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720199+0000","last_change":"2023-07-14T11:09:42.593674+0000","last_active":"2023-07-14T11:11:08.720199+0000","last_peered":"2023-07-14T11:11:08.720199+0000","last_clean":"2023-07-14T11:11:08.720199+0000","last_became_active":"2023-07-14T11:09:42.593345+0000","last_became_peered":"2023-07-14T11:09:42.593345+0000","last_unstale":"2023-07-14T11:11:08.720199+0000","last_undegraded":"2023-07-14T11:11:08.720199+0000","last_fullsized":"2023-07-14T11:11:08.720199+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T20:29:55.458801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,9,10],"acting":[3,9,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289387+0000","last_change":"2023-07-14T11:09:43.591176+0000","last_active":"2023-07-14T11:11:09.289387+0000","last_peered":"2023-07-14T11:11:09.289387+0000","last_clean":"2023-07-14T11:11:09.289387+0000","last_became_active":"2023-07-14T11:09:43.590854+0000","last_became_peered":"2023-07-14T11:09:43.590854+0000","last_unstale":"2023-07-14T11:11:09.289387+0000","last_undegraded":"2023-07-14T11:11:09.289387+0000","last_fullsized":"2023-07-14T11:11:09.289387+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:45:44.877318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,0,2],"acting":[11,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417886+0000","last_change":"2023-07-14T11:09:47.624868+0000","last_active":"2023-07-14T11:11:09.417886+0000","last_peered":"2023-07-14T11:11:09.417886+0000","last_clean":"2023-07-14T11:11:09.417886+0000","last_became_active":"2023-07-14T11:09:47.624450+0000","last_became_peered":"2023-07-14T11:09:47.624450+0000","last_unstale":"2023-07-14T11:11:09.417886+0000","last_undegraded":"2023-07-14T11:11:09.417886+0000","last_fullsized":"2023-07-14T11:11:09.417886+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:22:03.929374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,9,7,8],"acting":[4,9,7,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.a","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290180+0000","last_change":"2023-07-14T11:09:42.597295+0000","last_active":"2023-07-14T11:11:09.290180+0000","last_peered":"2023-07-14T11:11:09.290180+0000","last_clean":"2023-07-14T11:11:09.290180+0000","last_became_active":"2023-07-14T11:09:42.596695+0000","last_became_peered":"2023-07-14T11:09:42.596695+0000","last_unstale":"2023-07-14T11:11:09.290180+0000","last_undegraded":"2023-07-14T11:11:09.290180+0000","last_fullsized":"2023-07-14T11:11:09.290180+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:18:09.530818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,8],"acting":[2,3,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720181+0000","last_change":"2023-07-14T11:09:43.595953+0000","last_active":"2023-07-14T11:11:08.720181+0000","last_peered":"2023-07-14T11:11:08.720181+0000","last_clean":"2023-07-14T11:11:08.720181+0000","last_became_active":"2023-07-14T11:09:43.595620+0000","last_became_peered":"2023-07-14T11:09:43.595620+0000","last_unstale":"2023-07-14T11:11:08.720181+0000","last_undegraded":"2023-07-14T11:11:08.720181+0000","last_fullsized":"2023-07-14T11:11:08.720181+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T13:28:36.511332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,10],"acting":[3,0,10],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.417990+0000","last_change":"2023-07-14T11:09:47.737587+0000","last_active":"2023-07-14T11:11:09.417990+0000","last_peered":"2023-07-14T11:11:09.417990+0000","last_clean":"2023-07-14T11:11:09.417990+0000","last_became_active":"2023-07-14T11:09:47.736636+0000","last_became_peered":"2023-07-14T11:09:47.736636+0000","last_unstale":"2023-07-14T11:11:09.417990+0000","last_undegraded":"2023-07-14T11:11:09.417990+0000","last_fullsized":"2023-07-14T11:11:09.417990+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:34:29.312985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,10,3],"acting":[4,2,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.b","version":"90'4","reported_seq":35,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.722793+0000","last_change":"2023-07-14T11:09:42.600271+0000","last_active":"2023-07-14T11:11:08.722793+0000","last_peered":"2023-07-14T11:11:08.722793+0000","last_clean":"2023-07-14T11:11:08.722793+0000","last_became_active":"2023-07-14T11:09:42.599830+0000","last_became_peered":"2023-07-14T11:09:42.599830+0000","last_unstale":"2023-07-14T11:11:08.722793+0000","last_undegraded":"2023-07-14T11:11:08.722793+0000","last_fullsized":"2023-07-14T11:11:08.722793+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:34:50.906895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":4,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[8,2,7],"acting":[8,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":8,"acting_primary":8,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289355+0000","last_change":"2023-07-14T11:09:43.590704+0000","last_active":"2023-07-14T11:11:09.289355+0000","last_peered":"2023-07-14T11:11:09.289355+0000","last_clean":"2023-07-14T11:11:09.289355+0000","last_became_active":"2023-07-14T11:09:43.590394+0000","last_became_peered":"2023-07-14T11:09:43.590394+0000","last_unstale":"2023-07-14T11:11:09.289355+0000","last_undegraded":"2023-07-14T11:11:09.289355+0000","last_fullsized":"2023-07-14T11:11:09.289355+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:27:25.782530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.000947+0000","last_change":"2023-07-14T11:09:47.618537+0000","last_active":"2023-07-14T11:11:09.000947+0000","last_peered":"2023-07-14T11:11:09.000947+0000","last_clean":"2023-07-14T11:11:09.000947+0000","last_became_active":"2023-07-14T11:09:47.617981+0000","last_became_peered":"2023-07-14T11:09:47.617981+0000","last_unstale":"2023-07-14T11:11:09.000947+0000","last_undegraded":"2023-07-14T11:11:09.000947+0000","last_fullsized":"2023-07-14T11:11:09.000947+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:31:40.967816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6,7],"acting":[1,5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.c","version":"90'5","reported_seq":132,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:50.566007+0000","last_change":"2023-07-14T11:09:42.597566+0000","last_active":"2023-07-14T11:11:50.566007+0000","last_peered":"2023-07-14T11:11:50.566007+0000","last_clean":"2023-07-14T11:11:50.566007+0000","last_became_active":"2023-07-14T11:09:42.596763+0000","last_became_peered":"2023-07-14T11:09:42.596763+0000","last_unstale":"2023-07-14T11:11:50.566007+0000","last_undegraded":"2023-07-14T11:11:50.566007+0000","last_fullsized":"2023-07-14T11:11:50.566007+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:04:28.815042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1122,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":97,"num_read_kb":2,"num_write":5,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,11,0],"acting":[2,11,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289850+0000","last_change":"2023-07-14T11:09:43.595333+0000","last_active":"2023-07-14T11:11:09.289850+0000","last_peered":"2023-07-14T11:11:09.289850+0000","last_clean":"2023-07-14T11:11:09.289850+0000","last_became_active":"2023-07-14T11:09:43.594813+0000","last_became_peered":"2023-07-14T11:09:43.594813+0000","last_unstale":"2023-07-14T11:11:09.289850+0000","last_undegraded":"2023-07-14T11:11:09.289850+0000","last_fullsized":"2023-07-14T11:11:09.289850+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:30:10.951768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,11],"acting":[7,1,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.288876+0000","last_change":"2023-07-14T11:09:47.621866+0000","last_active":"2023-07-14T11:11:09.288876+0000","last_peered":"2023-07-14T11:11:09.288876+0000","last_clean":"2023-07-14T11:11:09.288876+0000","last_became_active":"2023-07-14T11:09:47.621326+0000","last_became_peered":"2023-07-14T11:09:47.621326+0000","last_unstale":"2023-07-14T11:11:09.288876+0000","last_undegraded":"2023-07-14T11:11:09.288876+0000","last_fullsized":"2023-07-14T11:11:09.288876+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:14:35.079631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,9,1,4],"acting":[0,9,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.d","version":"90'3","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720056+0000","last_change":"2023-07-14T11:09:42.595226+0000","last_active":"2023-07-14T11:11:08.720056+0000","last_peered":"2023-07-14T11:11:08.720056+0000","last_clean":"2023-07-14T11:11:08.720056+0000","last_became_active":"2023-07-14T11:09:42.594900+0000","last_became_peered":"2023-07-14T11:09:42.594900+0000","last_unstale":"2023-07-14T11:11:08.720056+0000","last_undegraded":"2023-07-14T11:11:08.720056+0000","last_fullsized":"2023-07-14T11:11:08.720056+0000","mapping_epoch":81,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":82,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T12:24:42.663850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,10,3],"acting":[5,10,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289490+0000","last_change":"2023-07-14T11:09:43.590527+0000","last_active":"2023-07-14T11:11:09.289490+0000","last_peered":"2023-07-14T11:11:09.289490+0000","last_clean":"2023-07-14T11:11:09.289490+0000","last_became_active":"2023-07-14T11:09:43.590163+0000","last_became_peered":"2023-07-14T11:09:43.590163+0000","last_unstale":"2023-07-14T11:11:09.289490+0000","last_undegraded":"2023-07-14T11:11:09.289490+0000","last_fullsized":"2023-07-14T11:11:09.289490+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:53:58.994547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719226+0000","last_change":"2023-07-14T11:09:47.617740+0000","last_active":"2023-07-14T11:11:08.719226+0000","last_peered":"2023-07-14T11:11:08.719226+0000","last_clean":"2023-07-14T11:11:08.719226+0000","last_became_active":"2023-07-14T11:09:47.617323+0000","last_became_peered":"2023-07-14T11:09:47.617323+0000","last_unstale":"2023-07-14T11:11:08.719226+0000","last_undegraded":"2023-07-14T11:11:08.719226+0000","last_fullsized":"2023-07-14T11:11:08.719226+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":86,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:46.586971+0000","last_clean_scrub_stamp":"2023-07-14T11:09:46.586971+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T18:15:14.394847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7,1],"acting":[5,2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.e","version":"90'5","reported_seq":45,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719234+0000","last_change":"2023-07-14T11:10:07.827927+0000","last_active":"2023-07-14T11:11:08.719234+0000","last_peered":"2023-07-14T11:11:08.719234+0000","last_clean":"2023-07-14T11:11:08.719234+0000","last_became_active":"2023-07-14T11:10:07.824694+0000","last_became_peered":"2023-07-14T11:10:07.824694+0000","last_unstale":"2023-07-14T11:11:08.719234+0000","last_undegraded":"2023-07-14T11:11:08.719234+0000","last_fullsized":"2023-07-14T11:11:08.719234+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":81,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:41.551931+0000","last_clean_scrub_stamp":"2023-07-14T11:09:41.551931+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:47:15.179813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289653+0000","last_change":"2023-07-14T11:09:43.585123+0000","last_active":"2023-07-14T11:11:09.289653+0000","last_peered":"2023-07-14T11:11:09.289653+0000","last_clean":"2023-07-14T11:11:09.289653+0000","last_became_active":"2023-07-14T11:09:43.584780+0000","last_became_peered":"2023-07-14T11:09:43.584780+0000","last_unstale":"2023-07-14T11:11:09.289653+0000","last_undegraded":"2023-07-14T11:11:09.289653+0000","last_fullsized":"2023-07-14T11:11:09.289653+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:11:51.768155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,11,8],"acting":[7,11,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289543+0000","last_change":"2023-07-14T11:09:43.590544+0000","last_active":"2023-07-14T11:11:09.289543+0000","last_peered":"2023-07-14T11:11:09.289543+0000","last_clean":"2023-07-14T11:11:09.289543+0000","last_became_active":"2023-07-14T11:09:43.590241+0000","last_became_peered":"2023-07-14T11:09:43.590241+0000","last_unstale":"2023-07-14T11:11:09.289543+0000","last_undegraded":"2023-07-14T11:11:09.289543+0000","last_fullsized":"2023-07-14T11:11:09.289543+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:14.076462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,2,5],"acting":[11,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290470+0000","last_change":"2023-07-14T11:09:43.598457+0000","last_active":"2023-07-14T11:11:09.290470+0000","last_peered":"2023-07-14T11:11:09.290470+0000","last_clean":"2023-07-14T11:11:09.290470+0000","last_became_active":"2023-07-14T11:09:43.597904+0000","last_became_peered":"2023-07-14T11:09:43.597904+0000","last_unstale":"2023-07-14T11:11:09.290470+0000","last_undegraded":"2023-07-14T11:11:09.290470+0000","last_fullsized":"2023-07-14T11:11:09.290470+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:22:58.418691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290343+0000","last_change":"2023-07-14T11:09:43.596270+0000","last_active":"2023-07-14T11:11:09.290343+0000","last_peered":"2023-07-14T11:11:09.290343+0000","last_clean":"2023-07-14T11:11:09.290343+0000","last_became_active":"2023-07-14T11:09:43.595944+0000","last_became_peered":"2023-07-14T11:09:43.595944+0000","last_unstale":"2023-07-14T11:11:09.290343+0000","last_undegraded":"2023-07-14T11:11:09.290343+0000","last_fullsized":"2023-07-14T11:11:09.290343+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:07:35.545905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":29,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.289064+0000","last_change":"2023-07-14T11:09:43.591943+0000","last_active":"2023-07-14T11:11:09.289064+0000","last_peered":"2023-07-14T11:11:09.289064+0000","last_clean":"2023-07-14T11:11:09.289064+0000","last_became_active":"2023-07-14T11:09:43.591610+0000","last_became_peered":"2023-07-14T11:09:43.591610+0000","last_unstale":"2023-07-14T11:11:09.289064+0000","last_undegraded":"2023-07-14T11:11:09.289064+0000","last_fullsized":"2023-07-14T11:11:09.289064+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:45:00.837634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":35,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719320+0000","last_change":"2023-07-14T11:10:08.143146+0000","last_active":"2023-07-14T11:11:08.719320+0000","last_peered":"2023-07-14T11:11:08.719320+0000","last_clean":"2023-07-14T11:11:08.719320+0000","last_became_active":"2023-07-14T11:10:08.142585+0000","last_became_peered":"2023-07-14T11:10:08.142585+0000","last_unstale":"2023-07-14T11:11:08.719320+0000","last_undegraded":"2023-07-14T11:11:08.719320+0000","last_fullsized":"2023-07-14T11:11:08.719320+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T23:03:36.474008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,9,0],"acting":[6,9,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":34,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.291336+0000","last_change":"2023-07-14T11:10:07.744121+0000","last_active":"2023-07-14T11:11:09.291336+0000","last_peered":"2023-07-14T11:11:09.291336+0000","last_clean":"2023-07-14T11:11:09.291336+0000","last_became_active":"2023-07-14T11:10:07.743654+0000","last_became_peered":"2023-07-14T11:10:07.743654+0000","last_unstale":"2023-07-14T11:11:09.291336+0000","last_undegraded":"2023-07-14T11:11:09.291336+0000","last_fullsized":"2023-07-14T11:11:09.291336+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T19:26:26.293702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,8,9],"acting":[11,8,9],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290764+0000","last_change":"2023-07-14T11:09:43.593880+0000","last_active":"2023-07-14T11:11:09.290764+0000","last_peered":"2023-07-14T11:11:09.290764+0000","last_clean":"2023-07-14T11:11:09.290764+0000","last_became_active":"2023-07-14T11:09:43.593481+0000","last_became_peered":"2023-07-14T11:09:43.593481+0000","last_unstale":"2023-07-14T11:11:09.290764+0000","last_undegraded":"2023-07-14T11:11:09.290764+0000","last_fullsized":"2023-07-14T11:11:09.290764+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:08:45.932302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,8,1],"acting":[2,8,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":15,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290111+0000","last_change":"2023-07-14T11:10:07.823702+0000","last_active":"2023-07-14T11:11:09.290111+0000","last_peered":"2023-07-14T11:11:09.290111+0000","last_clean":"2023-07-14T11:11:09.290111+0000","last_became_active":"2023-07-14T11:10:07.823349+0000","last_became_peered":"2023-07-14T11:10:07.823349+0000","last_unstale":"2023-07-14T11:11:09.290111+0000","last_undegraded":"2023-07-14T11:11:09.290111+0000","last_fullsized":"2023-07-14T11:11:09.290111+0000","mapping_epoch":89,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":90,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T11:58:00.414644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[9,4,2],"acting":[9,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":9,"acting_primary":9,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719489+0000","last_change":"2023-07-14T11:09:43.583012+0000","last_active":"2023-07-14T11:11:08.719489+0000","last_peered":"2023-07-14T11:11:08.719489+0000","last_clean":"2023-07-14T11:11:08.719489+0000","last_became_active":"2023-07-14T11:09:43.581922+0000","last_became_peered":"2023-07-14T11:09:43.581922+0000","last_unstale":"2023-07-14T11:11:08.719489+0000","last_undegraded":"2023-07-14T11:11:08.719489+0000","last_fullsized":"2023-07-14T11:11:08.719489+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T14:36:03.664035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,9,3],"acting":[5,9,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.720295+0000","last_change":"2023-07-14T11:09:43.596676+0000","last_active":"2023-07-14T11:11:08.720295+0000","last_peered":"2023-07-14T11:11:08.720295+0000","last_clean":"2023-07-14T11:11:08.720295+0000","last_became_active":"2023-07-14T11:09:43.596332+0000","last_became_peered":"2023-07-14T11:09:43.596332+0000","last_unstale":"2023-07-14T11:11:08.720295+0000","last_undegraded":"2023-07-14T11:11:08.720295+0000","last_fullsized":"2023-07-14T11:11:08.720295+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T21:05:07.837607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,11],"acting":[3,2,11],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.001328+0000","last_change":"2023-07-14T11:09:43.597111+0000","last_active":"2023-07-14T11:11:09.001328+0000","last_peered":"2023-07-14T11:11:09.001328+0000","last_clean":"2023-07-14T11:11:09.001328+0000","last_became_active":"2023-07-14T11:09:43.596788+0000","last_became_peered":"2023-07-14T11:09:43.596788+0000","last_unstale":"2023-07-14T11:11:09.001328+0000","last_undegraded":"2023-07-14T11:11:09.001328+0000","last_fullsized":"2023-07-14T11:11:09.001328+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T15:41:01.143943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:08.719197+0000","last_change":"2023-07-14T11:09:43.591575+0000","last_active":"2023-07-14T11:11:08.719197+0000","last_peered":"2023-07-14T11:11:08.719197+0000","last_clean":"2023-07-14T11:11:08.719197+0000","last_became_active":"2023-07-14T11:09:43.591212+0000","last_became_peered":"2023-07-14T11:09:43.591212+0000","last_unstale":"2023-07-14T11:11:08.719197+0000","last_undegraded":"2023-07-14T11:11:08.719197+0000","last_fullsized":"2023-07-14T11:11:08.719197+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T22:14:33.943135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":28,"reported_epoch":92,"state":"active+clean","last_fresh":"2023-07-14T11:11:09.290872+0000","last_change":"2023-07-14T11:09:43.587122+0000","last_active":"2023-07-14T11:11:09.290872+0000","last_peered":"2023-07-14T11:11:09.290872+0000","last_clean":"2023-07-14T11:11:09.290872+0000","last_became_active":"2023-07-14T11:09:43.586777+0000","last_became_peered":"2023-07-14T11:09:43.586777+0000","last_unstale":"2023-07-14T11:11:09.290872+0000","last_undegraded":"2023-07-14T11:11:09.290872+0000","last_fullsized":"2023-07-14T11:11:09.290872+0000","mapping_epoch":82,"log_start":"0'0","ondisk_log_start":"0'0","created":82,"last_epoch_clean":83,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-07-14T11:09:42.565544+0000","last_clean_scrub_stamp":"2023-07-14T11:09:42.565544+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-07-15T17:51:23.080257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[11,4,8],"acting":[11,4,8],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":11,"acting_primary":11,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":128,"acting":128,"num_store_stats":12},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":12},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":136,"num_read_kb":119,"num_write":181,"num_write_kb":2938,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4751360,"data_stored":4722944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137,"ondisk_log_size":137,"up":3,"acting":3,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":10282,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":1004,"num_read_kb":526,"num_write":211,"num_write_kb":69,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":66,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":393216,"data_stored":30846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185,"ondisk_log_size":185,"up":96,"acting":96,"num_store_stats":12}],"osd_stats":[{"osd":11,"up_from":79,"seq":339302416423,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27972,"kb_used_data":1020,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717180,"statfs":{"total":95995035648,"available":95966392320,"internally_reserved":0,"allocated":1044480,"data_stored":509181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.54800000000000004,"15min":0.55600000000000005},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.79400000000000004,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.439,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83099999999999996,"5min":1.202,"15min":1.202},"last":0.61899999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.64400000000000002,"15min":0.64100000000000001},"min":{"1min":0.52900000000000003,"5min":0.443,"15min":0.443},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.63100000000000001,"15min":0.627},"min":{"1min":0.49099999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.73499999999999999,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.379},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.35399999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.74199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.67100000000000004,"15min":0.67500000000000004},"min":{"1min":0.496,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.81299999999999994,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.68300000000000005,"15min":0.68100000000000005},"min":{"1min":0.56200000000000006,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.85099999999999998,"5min":1.05,"15min":1.05},"last":0.70399999999999996}]},{"osd":5,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40400000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79200000000000004,"5min":1.254,"15min":1.254},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.61099999999999999,"15min":0.61299999999999999},"min":{"1min":0.42599999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84599999999999997,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.73799999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.71199999999999997,"15min":0.71299999999999997},"min":{"1min":0.495,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.3220000000000001,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73799999999999999,"15min":0.74099999999999999},"min":{"1min":0.53600000000000003,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.92900000000000005,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.71799999999999997}]},{"osd":7,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.72799999999999998,"15min":0.72599999999999998},"min":{"1min":0.59699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.91400000000000003,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"min":{"1min":0.57499999999999996,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.89500000000000002,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.80400000000000005}]},{"osd":8,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.04,"5min":1.04,"15min":1.04},"last":0.754},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"min":{"1min":0.53700000000000003,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.0629999999999999,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.84099999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78600000000000003,"15min":0.78700000000000003},"min":{"1min":0.61799999999999999,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":0.95299999999999996,"5min":1.6950000000000001,"15min":1.6950000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":0.81999999999999995,"5min":0.80400000000000005,"15min":0.80200000000000005},"min":{"1min":0.58499999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4510000000000001,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.76900000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.82499999999999996,"15min":0.82199999999999995},"min":{"1min":0.64800000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.0149999999999999,"5min":1.411,"15min":1.411},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"min":{"1min":0.66700000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":0.99299999999999999,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.82499999999999996}]}]},{"osd":10,"up_from":72,"seq":309237645354,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28596,"kb_used_data":1644,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716556,"statfs":{"total":95995035648,"available":95965753344,"internally_reserved":0,"allocated":1683456,"data_stored":1104145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,8,9,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.44400000000000001,"5min":0.37,"15min":0.37},"max":{"1min":0.82799999999999996,"5min":3.9740000000000002,"15min":3.9740000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.77700000000000002,"15min":0.79500000000000004},"min":{"1min":0.54500000000000004,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.90500000000000003,"5min":4.0190000000000001,"15min":4.0190000000000001},"last":0.49199999999999999}]},{"osd":1,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.46000000000000002,"15min":0.45600000000000002},"min":{"1min":0.33100000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.59099999999999997,"5min":1.9419999999999999,"15min":1.9419999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.85399999999999998,"5min":1.7210000000000001,"15min":1.7210000000000001},"last":0.436}]},{"osd":2,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.65800000000000003,"15min":0.64400000000000002},"min":{"1min":0.47199999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.91400000000000003,"5min":1.0880000000000001,"15min":1.0880000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.66300000000000003,"15min":0.66700000000000004},"min":{"1min":0.44400000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.80000000000000004,"5min":1.796,"15min":1.796},"last":0.52100000000000002}]},{"osd":3,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.80700000000000005,"15min":0.88100000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69599999999999995,"5min":4.8230000000000004,"15min":4.8230000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.66500000000000004,"15min":0.67300000000000004},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.85199999999999998,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.60699999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57299999999999995,"15min":0.56299999999999994},"min":{"1min":0.41199999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.85499999999999998,"5min":1.839,"15min":1.839},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.626,"5min":0.57599999999999996,"15min":0.56200000000000006},"min":{"1min":0.39900000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.81000000000000005,"5min":2.0499999999999998,"15min":2.0499999999999998},"last":0.53500000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.73399999999999999,"15min":0.72799999999999998},"min":{"1min":0.60099999999999998,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.90900000000000003,"5min":1.165,"15min":1.165},"last":0.624},{"interface":"front","average":{"1min":0.746,"5min":0.76400000000000001,"15min":0.76900000000000002},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":0.90100000000000002,"5min":1.9850000000000001,"15min":1.9850000000000001},"last":0.74199999999999999}]},{"osd":6,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.77100000000000002,"15min":0.77900000000000003},"min":{"1min":0.621,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.88100000000000001,"5min":1.897,"15min":1.897},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.747,"5min":0.876,"15min":0.93000000000000005},"min":{"1min":0.55500000000000005,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96499999999999997,"5min":4.4900000000000002,"15min":4.4900000000000002},"last":0.72699999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:09:35 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.34100000000000003}]},{"osd":8,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.81100000000000005,"5min":0.78600000000000003,"15min":0.78400000000000003},"min":{"1min":0.63700000000000001,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.02,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.83699999999999997,"15min":0.84999999999999998},"min":{"1min":0.58699999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.999,"5min":1.6140000000000001,"15min":1.6140000000000001},"last":0.69999999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.84199999999999997,"15min":0.84599999999999997},"min":{"1min":0.64100000000000001,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.0169999999999999,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.871,"15min":0.88700000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.98799999999999999,"5min":1.7170000000000001,"15min":1.7170000000000001},"last":0.67200000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.84499999999999997,"15min":0.85299999999999998},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":0.98699999999999999,"5min":2.1440000000000001,"15min":2.1440000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.83199999999999996,"5min":0.99199999999999999,"15min":1.0189999999999999},"min":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.048,"5min":4.8609999999999998,"15min":4.8609999999999998},"last":0.81399999999999995}]}]},{"osd":9,"up_from":67,"seq":287762808878,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.504,"15min":0.52600000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72299999999999998,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.59099999999999997,"15min":0.621},"min":{"1min":0.34999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.98199999999999998,"5min":1.488,"15min":1.488},"last":0.38500000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.75,"15min":0.76500000000000001},"min":{"1min":0.48799999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.302,"5min":1.754,"15min":1.754},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69399999999999995,"15min":0.70599999999999996},"min":{"1min":0.437,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.214,"5min":1.214,"15min":1.214},"last":0.55500000000000005}]},{"osd":2,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.72899999999999998,"15min":0.75600000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.103,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.71799999999999997,"15min":0.746},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.73799999999999999,"15min":0.73999999999999999},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":5.3109999999999999,"5min":5.3109999999999999,"15min":5.3109999999999999},"last":0.62},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.68000000000000005,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.8100000000000001,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.751,"15min":0.755},"min":{"1min":0.59099999999999997,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.214,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.81100000000000005,"15min":0.81699999999999995},"min":{"1min":0.64500000000000002,"5min":0.498,"15min":0.498},"max":{"1min":1.379,"5min":1.379,"15min":1.379},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.83099999999999996,"15min":0.86299999999999999},"min":{"1min":0.55100000000000005,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":0.98199999999999998,"5min":1.242,"15min":1.242},"last":0.497},{"interface":"front","average":{"1min":0.76300000000000001,"5min":0.80000000000000004,"15min":0.83499999999999996},"min":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.097,"5min":1.258,"15min":1.258},"last":0.47199999999999998}]},{"osd":6,"last update":"Fri Jul 14 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.359,"5min":1.359,"15min":1.359},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4159999999999999,"5min":1.4159999999999999,"15min":1.4159999999999999},"last":0.39200000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":0.95799999999999996,"15min":0.95599999999999996},"min":{"1min":0.60799999999999998,"5min":0.495,"15min":0.495},"max":{"1min":5.2089999999999996,"5min":5.2089999999999996,"15min":5.2089999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.92200000000000004,"15min":0.96499999999999997},"min":{"1min":0.621,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.5269999999999999,"5min":1.9119999999999999,"15min":1.9119999999999999},"last":0.71099999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":0.87,"15min":0.90500000000000003},"min":{"1min":0.60199999999999998,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.2609999999999999,"5min":1.331,"15min":1.331},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.874,"5min":0.95199999999999996,"15min":0.99199999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.196,"5min":1.5329999999999999,"15min":1.5329999999999999},"last":0.53700000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":0.875,"15min":0.90400000000000003},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":1.298,"5min":1.7310000000000001,"15min":1.7310000000000001},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.137,"5min":1.002,"15min":0.98599999999999999},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":5.3579999999999997,"5min":5.3579999999999997,"15min":5.3579999999999997},"last":0.68200000000000005}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.82799999999999996,"15min":0.82499999999999996},"min":{"1min":0.59599999999999997,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.635,"5min":1.635,"15min":1.635},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.84599999999999997,"5min":0.877,"15min":0.88200000000000001},"min":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.5760000000000001,"5min":1.5760000000000001,"15min":1.5760000000000001},"last":0.64000000000000001}]}]},{"osd":8,"up_from":59,"seq":253403070515,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27992,"kb_used_data":1040,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717160,"statfs":{"total":95995035648,"available":95966371840,"internally_reserved":0,"allocated":1064960,"data_stored":509537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.83899999999999997,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59399999999999997,"15min":0.61399999999999999},"min":{"1min":0.41899999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.84899999999999998,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.55900000000000005,"15min":0.53600000000000003},"min":{"1min":0.51300000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78200000000000003,"5min":0.875,"15min":0.875},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.61499999999999999,"15min":0.60699999999999998},"min":{"1min":0.47199999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.79800000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.53900000000000003}]},{"osd":2,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.45800000000000002,"15min":0.46200000000000002},"min":{"1min":0.36199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.69599999999999995,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72299999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.71599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.621,"15min":0.63300000000000001},"min":{"1min":0.40300000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.93000000000000005,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.65500000000000003,"15min":0.66200000000000003},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.91800000000000004,"5min":1.079,"15min":1.079},"last":0.68700000000000006}]},{"osd":4,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.61899999999999999,"15min":0.61199999999999999},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.89700000000000002,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.64800000000000002,"15min":0.63600000000000001},"min":{"1min":0.56799999999999995,"5min":0.37,"15min":0.37},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.73699999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.38800000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.79800000000000004,"5min":1.052,"15min":1.052},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.377,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.82699999999999996,"5min":1.135,"15min":1.135},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.70599999999999996,"15min":0.70799999999999996},"min":{"1min":0.48899999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90100000000000002,"5min":1.008,"15min":1.008},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.73599999999999999,"15min":0.74199999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.76300000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:10:51 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.72499999999999998,"15min":0.71699999999999997},"min":{"1min":0.58199999999999996,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":1.028,"15min":1.028},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.73799999999999999,"15min":0.73199999999999998},"min":{"1min":0.54500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.99399999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.629}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76400000000000001,"15min":0.75600000000000001},"min":{"1min":0.61599999999999999,"5min":0.433,"15min":0.433},"max":{"1min":1.05,"5min":1.05,"15min":1.05},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.79100000000000004,"15min":0.77900000000000003},"min":{"1min":0.63100000000000001,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0309999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.81599999999999995}]},{"osd":10,"last update":"Fri Jul 14 11:11:41 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.75700000000000001,"15min":0.72899999999999998},"min":{"1min":0.65800000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.753},{"interface":"front","average":{"1min":0.876,"5min":0.78700000000000003,"15min":0.753},"min":{"1min":0.65100000000000002,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.081,"5min":1.085,"15min":1.085},"last":0.85399999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.81999999999999995,"15min":0.81399999999999995},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.0569999999999999,"5min":1.117,"15min":1.117},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.79200000000000004,"15min":0.78400000000000003},"min":{"1min":0.67200000000000004,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.83699999999999997}]}]},{"osd":7,"up_from":51,"seq":219043332151,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28568,"kb_used_data":1616,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716584,"statfs":{"total":95995035648,"available":95965782016,"internally_reserved":0,"allocated":1654784,"data_stored":1101130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.61099999999999999,"15min":0.60499999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.66400000000000003,"15min":0.65800000000000003},"min":{"1min":0.42399999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.98599999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.626}]},{"osd":1,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44400000000000001,"15min":0.443},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.66300000000000003,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45700000000000002,"15min":0.46300000000000002},"min":{"1min":0.34399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.58599999999999997,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59299999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.64100000000000001,"15min":0.65200000000000002},"min":{"1min":0.39400000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.92200000000000004,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63100000000000001,"15min":0.625},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.85799999999999998,"5min":0.872,"15min":0.872},"last":0.625}]},{"osd":3,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.61599999999999999,"15min":0.58799999999999997},"min":{"1min":0.44400000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.89200000000000002,"5min":3.6520000000000001,"15min":3.6520000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.627,"5min":0.63500000000000001,"15min":0.60299999999999998},"min":{"1min":0.40999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81899999999999995,"5min":3.7690000000000001,"15min":3.7690000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56000000000000005,"15min":0.55000000000000004},"min":{"1min":0.33000000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.78700000000000003,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.56799999999999995,"15min":0.57599999999999996},"min":{"1min":0.38600000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69099999999999995,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.76600000000000001}]},{"osd":5,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.70599999999999996,"15min":0.72199999999999998},"min":{"1min":0.55000000000000004,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.878,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.71199999999999997,"15min":0.71599999999999997},"min":{"1min":0.56899999999999995,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.88100000000000001,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.69099999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:30 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.74299999999999999,"15min":0.73799999999999999},"min":{"1min":0.47499999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.05,"5min":1.6259999999999999,"15min":1.6259999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.83199999999999996,"15min":0.75600000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":3.7810000000000001,"5min":7.3719999999999999,"15min":7.3719999999999999},"last":0.86199999999999999}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":0.76100000000000001,"15min":0.75900000000000001},"min":{"1min":0.65200000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.96599999999999997,"5min":1,"15min":1},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72199999999999998,"15min":0.71999999999999997},"min":{"1min":0.625,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.746}]},{"osd":9,"last update":"Fri Jul 14 11:11:19 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.83599999999999997,"15min":0.81799999999999995},"min":{"1min":0.61199999999999999,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":3.7109999999999999,"5min":3.7109999999999999,"15min":3.7109999999999999},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.1850000000000001,"5min":0.875,"15min":0.81899999999999995},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":7.2489999999999997,"5min":7.2489999999999997,"15min":7.2489999999999997},"last":0.89500000000000002}]},{"osd":10,"last update":"Fri Jul 14 11:11:39 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.72999999999999998,"15min":0.71999999999999997},"min":{"1min":0.56799999999999995,"5min":0.442,"15min":0.442},"max":{"1min":0.91500000000000004,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.78900000000000003,"15min":0.79000000000000004},"min":{"1min":0.59999999999999998,"5min":0.37,"15min":0.37},"max":{"1min":1.0580000000000001,"5min":1.216,"15min":1.216},"last":0.84599999999999997}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.83999999999999997,"15min":0.83599999999999997},"min":{"1min":0.70399999999999996,"5min":0.68200000000000005,"15min":0.68200000000000005},"max":{"1min":0.98599999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.877},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.85599999999999998,"15min":0.84999999999999998},"min":{"1min":0.79100000000000004,"5min":0.68000000000000005,"15min":0.68000000000000005},"max":{"1min":1.0129999999999999,"5min":1.0720000000000001,"15min":1.0720000000000001},"last":0.92000000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495677,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,5,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.51000000000000001,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":2.5190000000000001,"5min":2.5190000000000001,"15min":2.5190000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.64000000000000001,"15min":0.61399999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":0.47099999999999997}]},{"osd":1,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.0600000000000001,"5min":1.407,"15min":1.407},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.751}]},{"osd":2,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.76600000000000001,"15min":0.73699999999999999},"min":{"1min":0.60099999999999998,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.1299999999999999,"5min":1.1299999999999999,"15min":1.1299999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.75900000000000001,"15min":0.72099999999999997},"min":{"1min":0.54700000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":4.8250000000000002},"last":0.68600000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.67100000000000004,"15min":0.64300000000000002},"min":{"1min":0.46700000000000003,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":2.5430000000000001,"5min":2.5430000000000001,"15min":2.5430000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.623,"15min":0.60399999999999998},"min":{"1min":0.43099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":2.4540000000000002,"5min":2.4540000000000002,"15min":2.4540000000000002},"last":0.621}]},{"osd":4,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.76600000000000001,"15min":0.747},"min":{"1min":0.438,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.753,"5min":0.77500000000000002,"15min":0.76100000000000001},"min":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.028,"5min":1.302,"15min":1.302},"last":0.73499999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.73399999999999999,"15min":0.74199999999999999},"min":{"1min":0.57499999999999996,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.84299999999999997,"15min":0.82399999999999995},"min":{"1min":0.61599999999999999,"5min":0.441,"15min":0.441},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.65600000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.84899999999999998,"15min":0.81000000000000005},"min":{"1min":0.51300000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.83099999999999996,"15min":0.81799999999999995},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.113,"5min":1.502,"15min":1.502},"last":0.97699999999999998}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.95599999999999996,"15min":0.97399999999999998},"min":{"1min":0.64800000000000002,"5min":0.63900000000000001,"15min":0.63900000000000001},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.93400000000000005,"15min":0.96599999999999997},"min":{"1min":0.62,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.2270000000000001,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":0.89800000000000002}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.98499999999999999,"15min":1.0149999999999999},"min":{"1min":0.55500000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":2.577,"5min":2.577,"15min":2.577},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.0009999999999999,"15min":1.0329999999999999},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":2.5630000000000002,"5min":2.5630000000000002,"15min":2.5630000000000002},"last":0.879}]},{"osd":10,"last update":"Fri Jul 14 11:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.96299999999999997,"15min":0.97899999999999998},"min":{"1min":0.71299999999999997,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":1.101,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.85199999999999998,"15min":0.86799999999999999},"min":{"1min":0.50800000000000001,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.998,"5min":1.3140000000000001,"15min":1.3140000000000001},"last":0.80000000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:11:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":1.0349999999999999,"15min":1.012},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":4.6769999999999996,"5min":4.6769999999999996,"15min":4.6769999999999996},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.84499999999999997,"5min":0.88300000000000001,"15min":0.88900000000000001},"min":{"1min":0.65300000000000002,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.101,"5min":1.248,"15min":1.248},"last":0.85999999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574933,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":513793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.56100000000000005,"15min":0.53100000000000003},"min":{"1min":0.432,"5min":0.253,"15min":0.253},"max":{"1min":1.9610000000000001,"5min":1.9610000000000001,"15min":1.9610000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.60299999999999998,"15min":0.53100000000000003},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":0.66500000000000004}]},{"osd":2,"last update":"Fri Jul 14 11:11:38 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.66700000000000004,"15min":0.57099999999999995},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89200000000000002,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58999999999999997,"15min":0.52700000000000002},"min":{"1min":0.35099999999999998,"5min":0.34499999999999997,"15min":0.33900000000000002},"max":{"1min":0.82699999999999996,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.57699999999999996}]},{"osd":3,"last update":"Fri Jul 14 11:11:01 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":0.60299999999999998,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.863,"5min":1.863,"15min":1.863},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.71699999999999997,"15min":0.60899999999999999},"min":{"1min":0.45600000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.8420000000000001,"5min":5.3819999999999997,"15min":5.3819999999999997},"last":0.84699999999999998}]},{"osd":4,"last update":"Fri Jul 14 11:09:26 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45600000000000002,"15min":0.438},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.44600000000000001,"15min":0.434},"min":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68899999999999995,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49399999999999999}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.71699999999999997,"15min":0.66100000000000003},"min":{"1min":0.56100000000000005,"5min":0.38,"15min":0.38},"max":{"1min":0.96499999999999997,"5min":1.0629999999999999,"15min":1.0629999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64400000000000002,"15min":0.60599999999999998},"min":{"1min":0.48899999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.89600000000000002,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.69299999999999995}]},{"osd":6,"last update":"Fri Jul 14 11:11:07 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.72799999999999998,"15min":0.67900000000000005},"min":{"1min":0.48699999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.4100000000000001,"5min":2.6219999999999999,"15min":2.6219999999999999},"last":0.875},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.79500000000000004,"15min":0.73899999999999999},"min":{"1min":0.55600000000000005,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":2.0539999999999998,"5min":2.5710000000000002,"15min":2.5710000000000002},"last":1.028}]},{"osd":7,"last update":"Fri Jul 14 11:11:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.71299999999999997,"15min":0.67400000000000004},"min":{"1min":0.42499999999999999,"5min":0.316,"15min":0.316},"max":{"1min":1.0880000000000001,"5min":1.2110000000000001,"15min":1.2110000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":0.65900000000000003,"15min":0.629},"min":{"1min":0.40500000000000003,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.94099999999999995}]},{"osd":8,"last update":"Fri Jul 14 11:10:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":0.68999999999999995,"15min":0.65300000000000002},"min":{"1min":0.51900000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.01,"5min":1.04,"15min":1.04},"last":1.006},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.77800000000000002,"15min":0.72999999999999998},"min":{"1min":0.62,"5min":0.441,"15min":0.441},"max":{"1min":1.159,"5min":1.165,"15min":1.165},"last":0.97299999999999998}]},{"osd":9,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.85199999999999998,"15min":0.73699999999999999},"min":{"1min":0.57699999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":6.6980000000000004,"5min":6.6980000000000004,"15min":6.6980000000000004},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.3700000000000001,"5min":0.88100000000000001,"15min":0.77700000000000002},"min":{"1min":0.68400000000000005,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":6.593,"5min":6.593,"15min":6.593},"last":1.0549999999999999}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":0.83699999999999997,"15min":0.83199999999999996},"min":{"1min":0.45600000000000002,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":1.056,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.075},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.73699999999999999,"15min":0.72099999999999997},"min":{"1min":0.47399999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.0720000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.73799999999999999}]},{"osd":11,"last update":"Fri Jul 14 11:11:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.96499999999999997,"15min":0.97799999999999998},"min":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"max":{"1min":1.157,"5min":1.256,"15min":1.256},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.877,"15min":0.88800000000000001},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.83299999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738458,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27996,"kb_used_data":1044,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717156,"statfs":{"total":95995035648,"available":95966367744,"internally_reserved":0,"allocated":1069056,"data_stored":509894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.69699999999999995,"15min":0.58899999999999997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.9039999999999999,"5min":5.9039999999999999,"15min":5.9039999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66000000000000003,"15min":0.57699999999999996},"min":{"1min":0.42999999999999999,"5min":0.36799999999999999,"15min":0.36499999999999999},"max":{"1min":1.875,"5min":2.1349999999999998,"15min":2.1349999999999998},"last":0.54000000000000004}]},{"osd":2,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.66200000000000003,"15min":0.58499999999999996},"min":{"1min":0.46400000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.93400000000000005,"5min":2.528,"15min":2.528},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.66900000000000004,"15min":0.60599999999999998},"min":{"1min":0.44800000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.88,"5min":2.1280000000000001,"15min":2.1280000000000001},"last":0.76000000000000001}]},{"osd":3,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.71299999999999997,"15min":0.73299999999999998},"min":{"1min":0.437,"5min":0.433,"15min":0.433},"max":{"1min":0.82399999999999995,"5min":3.2599999999999998,"15min":3.2599999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.878,"15min":0.91500000000000004},"min":{"1min":0.45500000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.0920000000000001,"5min":6.1500000000000004,"15min":6.1500000000000004},"last":0.78600000000000003}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":0.70599999999999996,"15min":0.63600000000000001},"min":{"1min":0.47199999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.2829999999999999,"5min":2.48,"15min":2.48},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.80000000000000004,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":5.9610000000000003,"5min":5.9610000000000003,"15min":5.9610000000000003},"last":0.66700000000000004}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.81799999999999995,"15min":0.71499999999999997},"min":{"1min":0.55100000000000005,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.4019999999999999,"5min":6.0839999999999996,"15min":6.0839999999999996},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.75800000000000001,"15min":0.67000000000000004},"min":{"1min":0.62,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.02,"5min":2.5529999999999999,"15min":2.5529999999999999},"last":0.95499999999999996}]},{"osd":6,"last update":"Fri Jul 14 11:11:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.77800000000000002,"15min":0.71999999999999997},"min":{"1min":0.433,"5min":0.38,"15min":0.38},"max":{"1min":6.1870000000000003,"5min":6.1870000000000003,"15min":6.1870000000000003},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.74299999999999999,"15min":0.71899999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":3.7450000000000001,"5min":3.7450000000000001,"15min":3.7450000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.79500000000000004,"15min":0.79900000000000004},"min":{"1min":0.42599999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.0580000000000001,"5min":1.323,"15min":1.323},"last":0.62},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.88100000000000001,"15min":0.90100000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91000000000000003,"5min":1.9239999999999999,"15min":1.9239999999999999},"last":0.68799999999999994}]},{"osd":8,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":0.91100000000000003,"15min":0.88200000000000001},"min":{"1min":0.55700000000000005,"5min":0.496,"15min":0.496},"max":{"1min":6.0049999999999999,"5min":6.0049999999999999,"15min":6.0049999999999999},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.88100000000000001,"15min":0.89100000000000001},"min":{"1min":0.68600000000000005,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.83299999999999996}]},{"osd":9,"last update":"Fri Jul 14 11:11:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.82499999999999996,"15min":0.80500000000000005},"min":{"1min":0.50800000000000001,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":0.95399999999999996,"15min":0.91800000000000004},"min":{"1min":0.60299999999999998,"5min":0.437,"15min":0.437},"max":{"1min":6.1180000000000003,"5min":6.1180000000000003,"15min":6.1180000000000003},"last":0.93600000000000005}]},{"osd":10,"last update":"Fri Jul 14 11:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":0.876,"15min":0.86699999999999999},"min":{"1min":0.57799999999999996,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.456,"5min":2.085,"15min":2.085},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.85799999999999998,"15min":0.82799999999999996},"min":{"1min":0.69799999999999995,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.137,"5min":1.359,"15min":1.359},"last":0.92300000000000004}]},{"osd":11,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":0.95499999999999996,"15min":0.90000000000000002},"min":{"1min":0.77200000000000002,"5min":0.63800000000000001,"15min":0.63800000000000001},"max":{"1min":6.0460000000000003,"5min":6.0460000000000003,"15min":6.0460000000000003},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.026,"5min":0.88800000000000001,"15min":0.86499999999999999},"min":{"1min":0.752,"5min":0.57999999999999996,"15min":0.57999999999999996},"max":{"1min":2.1720000000000002,"5min":2.1720000000000002,"15min":2.1720000000000002},"last":0.91200000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411408,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28000,"kb_used_data":1048,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717152,"statfs":{"total":95995035648,"available":95966363648,"internally_reserved":0,"allocated":1073152,"data_stored":513692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,3,4,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.55700000000000005,"15min":0.54500000000000004},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.872,"5min":1.304,"15min":1.304},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.54200000000000004},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.33100000000000002},"max":{"1min":1.0229999999999999,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.44700000000000001}]},{"osd":1,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.59099999999999997,"15min":0.54800000000000004},"min":{"1min":0.42499999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.875,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.503},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.62,"15min":0.56599999999999995},"min":{"1min":0.54500000000000004,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Fri Jul 14 11:10:57 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.61899999999999999,"15min":0.60499999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.70599999999999996,"15min":0.64700000000000002},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":3.2210000000000001,"5min":3.2210000000000001,"15min":3.2210000000000001},"last":0.58199999999999996}]},{"osd":4,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.72099999999999997,"15min":0.72599999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.74399999999999999,"15min":0.745},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.97999999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.624}]},{"osd":5,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.502,"15min":0.46800000000000003},"min":{"1min":0.36899999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73499999999999999,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.52600000000000002,"15min":0.48399999999999999},"min":{"1min":0.41399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78800000000000003,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Fri Jul 14 11:11:10 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.623,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.69199999999999995,"15min":0.64800000000000002},"min":{"1min":0.57199999999999995,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.375,"5min":1.375,"15min":1.375},"last":0.83099999999999996}]},{"osd":7,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.79400000000000004,"15min":0.79200000000000004},"min":{"1min":0.59699999999999998,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":0.995,"5min":0.998,"15min":0.998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.76000000000000001,"5min":0.77200000000000002,"15min":0.77400000000000002},"min":{"1min":0.57699999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.94799999999999995,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.64900000000000002}]},{"osd":8,"last update":"Fri Jul 14 11:10:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.60399999999999998,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.61499999999999999,"15min":0.58899999999999997},"min":{"1min":0.51800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.69499999999999995}]},{"osd":9,"last update":"Fri Jul 14 11:11:16 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.76400000000000001,"15min":0.75},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.075,"5min":0.85699999999999998,"15min":0.81899999999999995},"min":{"1min":0.70299999999999996,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":3.27,"5min":3.27,"15min":3.27},"last":0.96299999999999997}]},{"osd":10,"last update":"Fri Jul 14 11:11:42 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.80800000000000005,"15min":0.78500000000000003},"min":{"1min":0.64900000000000002,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.0780000000000001,"5min":1.085,"15min":1.085},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.77100000000000002,"15min":0.74299999999999999},"min":{"1min":0.65600000000000003,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.879}]},{"osd":11,"last update":"Fri Jul 14 11:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.84299999999999997,"15min":0.84199999999999997},"min":{"1min":0.63300000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.159,"5min":1.159,"15min":1.159},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.79000000000000004,"15min":0.78600000000000003},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.032,"5min":1.032,"15min":1.032},"last":0.84999999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182474,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27984,"kb_used_data":1032,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717168,"statfs":{"total":95995035648,"available":95966380032,"internally_reserved":0,"allocated":1056768,"data_stored":508194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,4,5,6,7,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.48399999999999999,"15min":0.42099999999999999},"min":{"1min":0.379,"5min":0.252,"15min":0.252},"max":{"1min":1.4990000000000001,"5min":1.4990000000000001,"15min":1.4990000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.50800000000000001,"15min":0.44600000000000001},"min":{"1min":0.29499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.3120000000000001,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":0.48299999999999998}]},{"osd":1,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.61599999999999999,"15min":0.59399999999999997},"min":{"1min":0.374,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.591,"5min":1.591,"15min":1.591},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.73999999999999999,"5min":0.69299999999999995,"15min":0.60899999999999999},"min":{"1min":0.57799999999999996,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89400000000000002,"5min":3.8969999999999998,"15min":3.8969999999999998},"last":0.59099999999999997}]},{"osd":2,"last update":"Fri Jul 14 11:11:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":0.67300000000000004,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.3879999999999999,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.68899999999999995,"15min":0.65100000000000002},"min":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.56899999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":0.72799999999999998,"15min":0.65300000000000002},"min":{"1min":0.434,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.742,"5min":1.742,"15min":1.742},"last":0.87},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.77200000000000002,"15min":0.69999999999999996},"min":{"1min":0.61399999999999999,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.98199999999999998,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.877}]},{"osd":5,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.75900000000000001,"15min":0.73899999999999999},"min":{"1min":0.56499999999999995,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":0.92500000000000004,"5min":1.262,"15min":1.262},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.754,"15min":0.75},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":1.1200000000000001,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":0.622}]},{"osd":6,"last update":"Fri Jul 14 11:11:15 2023","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.748,"15min":0.72899999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.044,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.755,"5min":0.75,"15min":0.75900000000000001},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.60699999999999998}]},{"osd":7,"last update":"Fri Jul 14 11:11:37 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.81699999999999995,"15min":0.82699999999999996},"min":{"1min":0.53600000000000003,"5min":0.496,"15min":0.496},"max":{"1min":1.4670000000000001,"5min":1.5580000000000001,"15min":1.5580000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.83199999999999996,"15min":0.81599999999999995},"min":{"1min":0.53100000000000003,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.7050000000000001,"5min":1.7050000000000001,"15min":1.7050000000000001},"last":0.84499999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:08:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"min":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"max":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"last":0.93200000000000005},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.494,"5min":1.494,"15min":1.494},"last":0.96899999999999997}]},{"osd":9,"last update":"Fri Jul 14 11:11:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.89000000000000001,"15min":0.90800000000000003},"min":{"1min":0.46500000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.879,"5min":1.879,"15min":1.879},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.95299999999999996,"15min":0.98499999999999999},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.7490000000000001,"5min":1.7490000000000001,"15min":1.7490000000000001},"last":0.78600000000000003}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.98799999999999999,"15min":1.02},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":1.23,"5min":4.0300000000000002,"15min":4.0300000000000002},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.94299999999999995,"15min":0.96599999999999997},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.0009999999999999,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.85299999999999998}]},{"osd":11,"last update":"Fri Jul 14 11:11:46 2023","interfaces":[{"interface":"back","average":{"1min":0.81399999999999995,"5min":0.85799999999999998,"15min":0.86499999999999999},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.296,"5min":1.351,"15min":1.351},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.874,"5min":0.88200000000000001,"15min":0.88400000000000001},"min":{"1min":0.72399999999999998,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.014,"5min":1.163,"15min":1.163},"last":0.73799999999999999}]}]},{"osd":4,"up_from":33,"seq":141733920839,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27976,"kb_used_data":1024,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93717176,"statfs":{"total":95995035648,"available":95966388224,"internally_reserved":0,"allocated":1048576,"data_stored":508416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[1,2,3,5,6,7,8,9,10,11],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:09:23 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.57099999999999995,"15min":0.54900000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.59299999999999997,"15min":0.57499999999999996},"min":{"1min":0.48099999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.498}]},{"osd":1,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39100000000000001,"15min":0.36799999999999999},"min":{"1min":0.32800000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.47699999999999998,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.41999999999999998,"15min":0.39200000000000002},"min":{"1min":0.29299999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.59099999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52100000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60499999999999998,"15min":0.58099999999999996},"min":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.89400000000000002,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.64200000000000002,"15min":0.58899999999999997},"min":{"1min":0.47999999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.874,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.58599999999999997}]},{"osd":3,"last update":"Fri Jul 14 11:11:23 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63300000000000001,"15min":0.59099999999999997},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.841,"5min":1.841,"15min":1.841},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.622,"15min":0.59199999999999997},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.857,"5min":1.857,"15min":1.857},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.64300000000000002,"15min":0.59599999999999997},"min":{"1min":0.46999999999999997,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.69499999999999995,"15min":0.64600000000000002},"min":{"1min":0.60099999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.95199999999999996,"5min":1.0209999999999999,"15min":1.0209999999999999},"last":0.63400000000000001}]},{"osd":6,"last update":"Fri Jul 14 11:11:12 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.65300000000000002,"15min":0.63500000000000001},"min":{"1min":0.436,"5min":0.432,"15min":0.432},"max":{"1min":0.871,"5min":1.105,"15min":1.105},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.71499999999999997,"15min":0.68200000000000005},"min":{"1min":0.49199999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97599999999999998,"5min":1.137,"15min":1.137},"last":0.76600000000000001}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.65400000000000003,"15min":0.61799999999999999},"min":{"1min":0.45000000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.99299999999999999,"5min":1.087,"15min":1.087},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.628,"15min":0.59799999999999998},"min":{"1min":0.47599999999999998,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.82299999999999995,"5min":1.034,"15min":1.034},"last":0.83699999999999997}]},{"osd":8,"last update":"Fri Jul 14 11:10:49 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.73199999999999998,"15min":0.72799999999999998},"min":{"1min":0.54900000000000004,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.81000000000000005,"15min":0.80100000000000005},"min":{"1min":0.54800000000000004,"5min":0.52400000000000002,"15min":0.52400000000000002},"max":{"1min":1.0329999999999999,"5min":1.1539999999999999,"15min":1.1539999999999999},"last":0.877}]},{"osd":9,"last update":"Fri Jul 14 11:11:18 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.81499999999999995,"15min":0.84099999999999997},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":0.95299999999999996,"5min":1.1240000000000001,"15min":1.1240000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.81200000000000006,"15min":0.82399999999999995},"min":{"1min":0.64300000000000002,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.0589999999999999,"5min":1.123,"15min":1.123},"last":0.79600000000000004}]},{"osd":10,"last update":"Fri Jul 14 11:11:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.86499999999999999,"15min":0.88300000000000001},"min":{"1min":0.65200000000000002,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.075,"5min":1.212,"15min":1.212},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.752,"5min":0.79100000000000004,"15min":0.80100000000000005},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.93100000000000005,"5min":1.073,"15min":1.073},"last":0.90100000000000002}]},{"osd":11,"last update":"Fri Jul 14 11:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.91500000000000004,"15min":0.91400000000000003},"min":{"1min":0.72099999999999997,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.86799999999999999,"15min":0.86599999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.97999999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757314,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1628,"kb_used_omap":0,"kb_used_meta":26944,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1667072,"data_stored":1100904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":27590656},"hb_peers":[0,1,2,3,4,6,7,8,9,10],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.40500000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.746,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.54200000000000004,"15min":0.55400000000000005},"min":{"1min":0.36699999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.75700000000000001,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52800000000000002}]},{"osd":1,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59099999999999997,"15min":0.58499999999999996},"min":{"1min":0.42699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.878,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.749},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.61799999999999999,"15min":0.59599999999999997},"min":{"1min":0.53000000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.83799999999999997,"5min":1.466,"15min":1.466},"last":0.64700000000000002}]},{"osd":2,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.34300000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68999999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.45200000000000001,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.60499999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.60399999999999998}]},{"osd":3,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.60099999999999998},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41799999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78600000000000003,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.69299999999999995}]},{"osd":4,"last update":"Fri Jul 14 11:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.71299999999999997,"15min":0.65900000000000003},"min":{"1min":0.49199999999999999,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.88300000000000001,"5min":4.9340000000000002,"15min":4.9340000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.69599999999999995,"15min":0.64600000000000002},"min":{"1min":0.51700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.90900000000000003,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Fri Jul 14 11:11:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65300000000000002,"15min":0.66300000000000003},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82199999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.68600000000000005,"15min":0.68999999999999995},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.161,"5min":1.161,"15min":1.161},"last":0.77100000000000002}]},{"osd":7,"last update":"Fri Jul 14 11:11:32 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.70699999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.93100000000000005,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.745,"5min":0.73599999999999999,"15min":0.73999999999999999},"min":{"1min":0.54900000000000004,"5min":0.496,"15min":0.496},"max":{"1min":0.93200000000000005,"5min":1.528,"15min":1.528},"last":0.91900000000000004}]},{"osd":8,"last update":"Fri Jul 14 11:10:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.63700000000000001,"15min":0.63200000000000001},"min":{"1min":0.38400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.373,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86799999999999999,"5min":0.878,"15min":0.878},"last":0.90500000000000003}]},{"osd":9,"last update":"Fri Jul 14 11:11:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.75600000000000001,"15min":0.74399999999999999},"min":{"1min":0.49299999999999999,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.488,"5min":1.488,"15min":1.488},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.80800000000000005,"5min":0.77200000000000002,"15min":0.76000000000000001},"min":{"1min":0.59199999999999997,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.88400000000000001}]},{"osd":10,"last update":"Fri Jul 14 11:11:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":0.75600000000000001,"15min":0.73599999999999999},"min":{"1min":0.61899999999999999,"5min":0.439,"15min":0.439},"max":{"1min":0.95299999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.98599999999999999},{"interface":"front","average":{"1min":0.78300000000000003,"5min":0.91000000000000003,"15min":0.96199999999999997},"min":{"1min":0.59499999999999997,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.93100000000000005,"5min":4.9900000000000002,"15min":4.9900000000000002},"last":0.85099999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":40960,"data_stored":2130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":5928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":2772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":2998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":1500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":36864,"data_stored":1122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":57344,"data_stored":5740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":8,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":9,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":10,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":11,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-07-14T11:11:55.350 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:55.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 -- 172.21.15.40:0/3002061461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec0ffb00 msgr2=0x7f9bec0fff10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/3002061461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec0ffb00 0x7f9bec0fff10 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f9bdc0098f0 tx=0x7f9bdc009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:55.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 -- 172.21.15.40:0/3002061461 shutdown_connections 2023-07-14T11:11:55.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/3002061461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 0x7f9bec107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/3002061461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec0ffb00 0x7f9bec0fff10 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/3002061461 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec100cf0 0x7f9bec101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 -- 172.21.15.40:0/3002061461 >> 172.21.15.40:0/3002061461 conn(0x7f9bec0fb0e0 msgr2=0x7f9bec0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:55.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 -- 172.21.15.40:0/3002061461 shutdown_connections 2023-07-14T11:11:55.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.582+0000 7f9bf27ff700 1 -- 172.21.15.40:0/3002061461 wait complete. 2023-07-14T11:11:55.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.583+0000 7f9bf27ff700 1 Processor -- start 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.583+0000 7f9bf27ff700 1 -- start start 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.583+0000 7f9bf27ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.583+0000 7f9bf27ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec100cf0 0x7f9bec197f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bf27ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 0x7f9bec19efd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bf27ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bec19f5a0 con 0x7f9bec0ffb00 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bf27ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bec19f6e0 con 0x7f9bec100cf0 2023-07-14T11:11:55.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bf27ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bec19f870 con 0x7f9bec101680 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55592/0 (socket says 172.21.15.40:55592) 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9beb7fe700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec100cf0 0x7f9bec197f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 -- 172.21.15.40:0/2478448138 learned_addr learned my addr 172.21.15.40:0/2478448138 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bf0d9c700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 0x7f9bec19efd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 -- 172.21.15.40:0/2478448138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 msgr2=0x7f9bec19efd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 0x7f9bec19efd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 -- 172.21.15.40:0/2478448138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec100cf0 msgr2=0x7f9bec197f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec100cf0 0x7f9bec197f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 -- 172.21.15.40:0/2478448138 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9be00095f0 con 0x7f9bec0ffb00 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.584+0000 7f9bebfff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f9bdc000c00 tx=0x7f9bdc00b730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.585+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bdc01c070 con 0x7f9bec0ffb00 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.585+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bdc009520 con 0x7f9bec0ffb00 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.585+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bdc00f7b0 con 0x7f9bec0ffb00 2023-07-14T11:11:55.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.585+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bdc003a00 con 0x7f9bec0ffb00 2023-07-14T11:11:55.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.585+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bec19fde0 con 0x7f9bec0ffb00 2023-07-14T11:11:55.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.586+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9bec10b2d0 con 0x7f9bec0ffb00 2023-07-14T11:11:55.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.588+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f9bdc020030 con 0x7f9bec0ffb00 2023-07-14T11:11:55.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.589+0000 7f9be97fa700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 0x7f9bd40a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:55.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.589+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f9bdc0cdd20 con 0x7f9bec0ffb00 2023-07-14T11:11:55.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.589+0000 7f9beb7fe700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 0x7f9bd40a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:55.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.590+0000 7f9beb7fe700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 0x7f9bd40a3650 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f9be0005fd0 tx=0x7f9be0009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:55.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.593+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f9bdc097150 con 0x7f9bec0ffb00 2023-07-14T11:11:55.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.808+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f9bec1a0720 con 0x7f9bec0ffb00 2023-07-14T11:11:55.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.810+0000 7f9be97fa700 1 -- 172.21.15.40:0/2478448138 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7f9bdc09a770 con 0x7f9bec0ffb00 2023-07-14T11:11:55.813 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:55.813 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 msgr2=0x7f9bd40a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 0x7f9bd40a3650 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f9be0005fd0 tx=0x7f9be0009310 comp rx=0 tx=0).stop 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 msgr2=0x7f9bec197a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 crypto rx=0x7f9bdc000c00 tx=0x7f9bdc00b730 comp rx=0 tx=0).stop 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 shutdown_connections 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f9bd40a11a0 0x7f9bd40a3650 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bec101680 0x7f9bec19efd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bec0ffb00 0x7f9bec197a20 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 --2- 172.21.15.40:0/2478448138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bec100cf0 0x7f9bec197f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:55.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 >> 172.21.15.40:0/2478448138 conn(0x7f9bec0fb0e0 msgr2=0x7f9bec0fd2e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:55.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 shutdown_connections 2023-07-14T11:11:55.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:55.812+0000 7f9bf27ff700 1 -- 172.21.15.40:0/2478448138 wait complete. 2023-07-14T11:11:55.833 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '84', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 5, 'flags_names': 'hashpspool,ec_overwrites', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '88', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:11:55.834 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:11:56.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.038+0000 7fe4ccfff700 1 -- 172.21.15.40:0/3499171209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 msgr2=0x7fe4c80729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.038+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/3499171209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c80729a0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 crypto rx=0x7fe4b8009990 tx=0x7fe4b8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 -- 172.21.15.40:0/3499171209 shutdown_connections 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/3499171209 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c810c0c0 0x7fe4c810e4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/3499171209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c80729a0 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/3499171209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c8072f70 0x7fe4c80733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 -- 172.21.15.40:0/3499171209 >> 172.21.15.40:0/3499171209 conn(0x7fe4c80fb130 msgr2=0x7fe4c80fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:56.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 -- 172.21.15.40:0/3499171209 shutdown_connections 2023-07-14T11:11:56.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.039+0000 7fe4ccfff700 1 -- 172.21.15.40:0/3499171209 wait complete. 2023-07-14T11:11:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:55 smithi130 ceph-mon[129405]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:55 smithi130 ceph-mon[129405]: pgmap v346: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:55 smithi130 ceph-mon[129405]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 Processor -- start 2023-07-14T11:11:56.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 -- start start 2023-07-14T11:11:56.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c8072f70 0x7fe4c819be80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 0x7fe4c81a3430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c81a3a00 con 0x7fe4c8074550 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c81a3b40 con 0x7fe4c8072f70 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.040+0000 7fe4ccfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe4c81a3cd0 con 0x7fe4c810c0c0 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55620/0 (socket says 172.21.15.40:55620) 2023-07-14T11:11:56.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 -- 172.21.15.40:0/2606088324 learned_addr learned my addr 172.21.15.40:0/2606088324 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4c659c700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c8072f70 0x7fe4c819be80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4c6d9d700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 0x7fe4c81a3430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 -- 172.21.15.40:0/2606088324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 msgr2=0x7fe4c81a3430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 0x7fe4c81a3430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 -- 172.21.15.40:0/2606088324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c8072f70 msgr2=0x7fe4c819be80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c8072f70 0x7fe4c819be80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 -- 172.21.15.40:0/2606088324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe4b80095f0 con 0x7fe4c8074550 2023-07-14T11:11:56.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4c6d9d700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 0x7fe4c81a3430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bffff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fe4b000d980 tx=0x7fe4b000dc90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.041+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4b0004020 con 0x7fe4c8074550 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.042+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe4b0009730 con 0x7fe4c8074550 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.042+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe4c81a3fb0 con 0x7fe4c8074550 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.042+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe4b0003a30 con 0x7fe4c8074550 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.042+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe4c81a4430 con 0x7fe4c8074550 2023-07-14T11:11:56.048 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.043+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe4c8101c10 con 0x7fe4c8074550 2023-07-14T11:11:56.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:55 smithi040 ceph-mon[97954]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:55 smithi040 ceph-mon[97954]: pgmap v346: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:56.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:55 smithi040 ceph-mon[97954]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.045+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fe4b00098a0 con 0x7fe4c8074550 2023-07-14T11:11:56.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.046+0000 7fe4bf7fe700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 0x7fe4b40a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.046+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fe4b00cc980 con 0x7fe4c8074550 2023-07-14T11:11:56.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.046+0000 7fe4c659c700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 0x7fe4b40a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.050 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.047+0000 7fe4c659c700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 0x7fe4b40a3600 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe4b800b5c0 tx=0x7fe4b8005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.052 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.050+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fe4b0095d20 con 0x7fe4c8074550 2023-07-14T11:11:56.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:55 smithi045 ceph-mon[128791]: from='client.15468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:55 smithi045 ceph-mon[128791]: pgmap v346: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 5.9 KiB/s rd, 170 B/s wr, 10 op/s 2023-07-14T11:11:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:55 smithi045 ceph-mon[128791]: from='client.15474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:56.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.258+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7fe4c80611c0 con 0x7fe4c8074550 2023-07-14T11:11:56.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.258+0000 7fe4bf7fe700 1 -- 172.21.15.40:0/2606088324 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v92) v1 ==== 93+0+10 (secure 0 0 0) 0x7fe4b0099340 con 0x7fe4c8074550 2023-07-14T11:11:56.261 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:11:56.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.260+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 msgr2=0x7fe4b40a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 0x7fe4b40a3600 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe4b800b5c0 tx=0x7fe4b8005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 msgr2=0x7fe4c819c3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 crypto rx=0x7fe4b000d980 tx=0x7fe4b000dc90 comp rx=0 tx=0).stop 2023-07-14T11:11:56.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 shutdown_connections 2023-07-14T11:11:56.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fe4b40a1150 0x7fe4b40a3600 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe4c810c0c0 0x7fe4c81a3430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe4c8072f70 0x7fe4c819be80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 --2- 172.21.15.40:0/2606088324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe4c8074550 0x7fe4c819c3c0 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 >> 172.21.15.40:0/2606088324 conn(0x7fe4c80fb130 msgr2=0x7fe4c80fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:56.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 shutdown_connections 2023-07-14T11:11:56.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.261+0000 7fe4ccfff700 1 -- 172.21.15.40:0/2606088324 wait complete. 2023-07-14T11:11:56.282 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 -- 172.21.15.40:0/712511443 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0100ce0 msgr2=0x7fdbd0101130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/712511443 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0101130 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7fdbc4009990 tx=0x7fdbc4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 -- 172.21.15.40:0/712511443 shutdown_connections 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/712511443 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0101670 0x7fdbd0107f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/712511443 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 0x7fdbd00fff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/712511443 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0101130 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 -- 172.21.15.40:0/712511443 >> 172.21.15.40:0/712511443 conn(0x7fdbd00fb130 msgr2=0x7fdbd00fd540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:56.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 -- 172.21.15.40:0/712511443 shutdown_connections 2023-07-14T11:11:56.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.487+0000 7fdbd5dff700 1 -- 172.21.15.40:0/712511443 wait complete. 2023-07-14T11:11:56.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.488+0000 7fdbd5dff700 1 Processor -- start 2023-07-14T11:11:56.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.488+0000 7fdbd5dff700 1 -- start start 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 0x7fdbd0197a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0197fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbd019f5f0 con 0x7fdbd0100ce0 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbd019f730 con 0x7fdbd0101670 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbd5dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdbd019f8c0 con 0x7fdbd00ffaf0 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbcffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0197fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbcffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49172/0 (socket says 172.21.15.40:49172) 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbcffff700 1 -- 172.21.15.40:0/2408581632 learned_addr learned my addr 172.21.15.40:0/2408581632 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbcf7fe700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 0x7fdbd0197a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 -- 172.21.15.40:0/2408581632 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 msgr2=0x7fdbd0197a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 0x7fdbd0197a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 -- 172.21.15.40:0/2408581632 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 msgr2=0x7fdbd019f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbceffd700 1 -- 172.21.15.40:0/2408581632 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdbc8009c40 con 0x7fdbd0100ce0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.489+0000 7fdbcffff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbceffd700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0197fb0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fdbc400b620 tx=0x7fdbc4004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdbc401d070 con 0x7fdbd0100ce0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdbc40095f0 con 0x7fdbd0100ce0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdbc4021410 con 0x7fdbd0100ce0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdbc4017650 con 0x7fdbd0100ce0 2023-07-14T11:11:56.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.490+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdbd019fe30 con 0x7fdbd0100ce0 2023-07-14T11:11:56.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.491+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdbd010b310 con 0x7fdbd0100ce0 2023-07-14T11:11:56.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.493+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fdbc40217f0 con 0x7fdbd0100ce0 2023-07-14T11:11:56.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.494+0000 7fdbccff9700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 0x7fdbb80a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.494+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fdbc40cdff0 con 0x7fdbd0100ce0 2023-07-14T11:11:56.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.495+0000 7fdbcf7fe700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 0x7fdbb80a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.495+0000 7fdbcf7fe700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 0x7fdbb80a3680 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fdbd0100b40 tx=0x7fdbc0009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.498+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fdbc4097420 con 0x7fdbd0100ce0 2023-07-14T11:11:56.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.711+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fdbd01a0470 con 0x7fdbd0100ce0 2023-07-14T11:11:56.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.712+0000 7fdbccff9700 1 -- 172.21.15.40:0/2408581632 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v92) v1 ==== 104+0+11 (secure 0 0 0) 0x7fdbc409aa40 con 0x7fdbd0100ce0 2023-07-14T11:11:56.715 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:56.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 msgr2=0x7fdbb80a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 0x7fdbb80a3680 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7fdbd0100b40 tx=0x7fdbc0009260 comp rx=0 tx=0).stop 2023-07-14T11:11:56.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 msgr2=0x7fdbd0197fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0197fb0 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 crypto rx=0x7fdbc400b620 tx=0x7fdbc4004730 comp rx=0 tx=0).stop 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 shutdown_connections 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fdbb80a11d0 0x7fdbb80a3680 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdbd0101670 0x7fdbd019f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdbd00ffaf0 0x7fdbd0197a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 --2- 172.21.15.40:0/2408581632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdbd0100ce0 0x7fdbd0197fb0 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.714+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 >> 172.21.15.40:0/2408581632 conn(0x7fdbd00fb130 msgr2=0x7fdbd00fd360 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:56.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.715+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 shutdown_connections 2023-07-14T11:11:56.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.715+0000 7fdbd5dff700 1 -- 172.21.15.40:0/2408581632 wait complete. 2023-07-14T11:11:56.736 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:11:56.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 -- 172.21.15.40:0/3777820562 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 msgr2=0x7f33a0101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 --2- 172.21.15.40:0/3777820562 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0101120 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f338c0098f0 tx=0x7f338c009c00 comp rx=0 tx=0).stop 2023-07-14T11:11:56.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 -- 172.21.15.40:0/3777820562 shutdown_connections 2023-07-14T11:11:56.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 --2- 172.21.15.40:0/3777820562 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a0101660 0x7f33a0107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 --2- 172.21.15.40:0/3777820562 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a00ffae0 0x7f33a00ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 --2- 172.21.15.40:0/3777820562 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0101120 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 -- 172.21.15.40:0/3777820562 >> 172.21.15.40:0/3777820562 conn(0x7f33a00fb0e0 msgr2=0x7f33a00fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:56.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 -- 172.21.15.40:0/3777820562 shutdown_connections 2023-07-14T11:11:56.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.943+0000 7f33a71ff700 1 -- 172.21.15.40:0/3777820562 wait complete. 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.944+0000 7f33a71ff700 1 Processor -- start 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.944+0000 7f33a71ff700 1 -- start start 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0197ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a0101660 0x7f33a019f060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33a019f630 con 0x7f33a00ffae0 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33a019f770 con 0x7f33a0100cd0 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a71ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33a019f900 con 0x7f33a0101660 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55666/0 (socket says 172.21.15.40:55666) 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 -- 172.21.15.40:0/4080624639 learned_addr learned my addr 172.21.15.40:0/4080624639 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f339ffff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0197ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a579c700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a0101660 0x7f33a019f060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 -- 172.21.15.40:0/4080624639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a0101660 msgr2=0x7f33a019f060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a0101660 0x7f33a019f060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 -- 172.21.15.40:0/4080624639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 msgr2=0x7f33a0197ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0197ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.945+0000 7f33a4f9b700 1 -- 172.21.15.40:0/4080624639 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3398009d00 con 0x7f33a00ffae0 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f33a4f9b700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f339400e970 tx=0x7f339400ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3394009790 con 0x7f33a00ffae0 2023-07-14T11:11:56.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f338c009520 con 0x7f33a00ffae0 2023-07-14T11:11:56.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f339400c7e0 con 0x7f33a00ffae0 2023-07-14T11:11:56.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3394003c70 con 0x7f33a00ffae0 2023-07-14T11:11:56.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.946+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33a019ff60 con 0x7f33a00ffae0 2023-07-14T11:11:56.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.947+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f33a010b300 con 0x7f33a00ffae0 2023-07-14T11:11:56.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.949+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f339400cc50 con 0x7f33a00ffae0 2023-07-14T11:11:56.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.950+0000 7f339dffb700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 0x7f33900a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:56.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.951+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f3394014070 con 0x7f33a00ffae0 2023-07-14T11:11:56.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.951+0000 7f339ffff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 0x7f33900a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:56.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.951+0000 7f339ffff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 0x7f33900a3600 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f338c005e90 tx=0x7f338c019040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:56.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:56.954+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3394096210 con 0x7f33a00ffae0 2023-07-14T11:11:57.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:56 smithi130 ceph-mon[129405]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:57.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:56 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2478448138' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:57.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:56 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2606088324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:56 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2408581632' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:56 smithi040 ceph-mon[97954]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:57.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:56 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2478448138' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:57.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:56 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2606088324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:56 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2408581632' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:56 smithi045 ceph-mon[128791]: from='client.15480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-07-14T11:11:57.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:56 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2478448138' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:57.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:56 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2606088324' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:56 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2408581632' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:11:57.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.168+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f33a019fac0 con 0x7f33a00ffae0 2023-07-14T11:11:57.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.169+0000 7f339dffb700 1 -- 172.21.15.40:0/4080624639 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v92) v1 ==== 100+0+11 (secure 0 0 0) 0x7f3394003dd0 con 0x7f33a00ffae0 2023-07-14T11:11:57.171 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:11:57.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 msgr2=0x7f33900a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 0x7f33900a3600 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f338c005e90 tx=0x7f338c019040 comp rx=0 tx=0).stop 2023-07-14T11:11:57.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 msgr2=0x7f33a0197ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 crypto rx=0x7f339400e970 tx=0x7f339400ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:57.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 shutdown_connections 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f33900a1150 0x7f33900a3600 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f33a0101660 0x7f33a019f060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33a00ffae0 0x7f33a0197ab0 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 --2- 172.21.15.40:0/4080624639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f33a0100cd0 0x7f33a0197ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 >> 172.21.15.40:0/4080624639 conn(0x7f33a00fb0e0 msgr2=0x7f33a00fd340 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:57.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 shutdown_connections 2023-07-14T11:11:57.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.171+0000 7f33a71ff700 1 -- 172.21.15.40:0/4080624639 wait complete. 2023-07-14T11:11:57.192 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:11:57.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2481117050 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 msgr2=0x7f6c880729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2481117050 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c880729a0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f6c78009990 tx=0x7f6c78009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2481117050 shutdown_connections 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2481117050 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c881079a0 0x7f6c88109e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2481117050 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c880729a0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2481117050 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c88072f70 0x7f6c880733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2481117050 >> 172.21.15.40:0/2481117050 conn(0x7f6c880fb0e0 msgr2=0x7f6c880fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2481117050 shutdown_connections 2023-07-14T11:11:57.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.396+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2481117050 wait complete. 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.397+0000 7f6c8ddff700 1 Processor -- start 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 -- start start 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c88198020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 0x7f6c8819f090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c8819f660 con 0x7f6c88072f70 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c8819f7a0 con 0x7f6c88074550 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c8ddff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c8819f930 con 0x7f6c881079a0 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c877fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c877fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55674/0 (socket says 172.21.15.40:55674) 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c877fe700 1 -- 172.21.15.40:0/2985787946 learned_addr learned my addr 172.21.15.40:0/2985787946 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c86ffd700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c88198020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c87fff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 0x7f6c8819f090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.398+0000 7f6c877fe700 1 -- 172.21.15.40:0/2985787946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 msgr2=0x7f6c8819f090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c877fe700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 0x7f6c8819f090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c877fe700 1 -- 172.21.15.40:0/2985787946 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 msgr2=0x7f6c88198020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c877fe700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c88198020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c877fe700 1 -- 172.21.15.40:0/2985787946 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c780095f0 con 0x7f6c88072f70 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c87fff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 0x7f6c8819f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c86ffd700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c88198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:57.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c877fe700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f6c78004710 tx=0x7f6c78009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:57.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c7801d070 con 0x7f6c88072f70 2023-07-14T11:11:57.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c8819fbb0 con 0x7f6c88072f70 2023-07-14T11:11:57.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6c7800fbc0 con 0x7f6c88072f70 2023-07-14T11:11:57.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c78017650 con 0x7f6c88072f70 2023-07-14T11:11:57.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.399+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c8819ffd0 con 0x7f6c88072f70 2023-07-14T11:11:57.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.400+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c88106050 con 0x7f6c88072f70 2023-07-14T11:11:57.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.402+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f6c78021410 con 0x7f6c88072f70 2023-07-14T11:11:57.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.404+0000 7f6c84ff9700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 0x7f6c700a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.404+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f6c780ce260 con 0x7f6c88072f70 2023-07-14T11:11:57.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.405+0000 7f6c86ffd700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 0x7f6c700a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.405+0000 7f6c86ffd700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 0x7f6c700a3600 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f6c88074270 tx=0x7f6c7c009260 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:57.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.408+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6c78097690 con 0x7f6c88072f70 2023-07-14T11:11:57.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.619+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f6c880611c0 con 0x7f6c88072f70 2023-07-14T11:11:57.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.620+0000 7f6c84ff9700 1 -- 172.21.15.40:0/2985787946 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v92) v1 ==== 103+0+11 (secure 0 0 0) 0x7f6c7809acb0 con 0x7f6c88072f70 2023-07-14T11:11:57.623 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:11:57.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 msgr2=0x7f6c700a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 0x7f6c700a3600 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f6c88074270 tx=0x7f6c7c009260 comp rx=0 tx=0).stop 2023-07-14T11:11:57.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 msgr2=0x7f6c88197ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 crypto rx=0x7f6c78004710 tx=0x7f6c78009580 comp rx=0 tx=0).stop 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 shutdown_connections 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f6c700a1150 0x7f6c700a3600 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6c881079a0 0x7f6c8819f090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6c88072f70 0x7f6c88197ae0 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 --2- 172.21.15.40:0/2985787946 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6c88074550 0x7f6c88198020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.622+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 >> 172.21.15.40:0/2985787946 conn(0x7f6c880fb0e0 msgr2=0x7f6c880fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:57.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.623+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 shutdown_connections 2023-07-14T11:11:57.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.623+0000 7f6c8ddff700 1 -- 172.21.15.40:0/2985787946 wait complete. 2023-07-14T11:11:57.644 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:11:57.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.847+0000 7fc0523ff700 1 -- 172.21.15.40:0/540270094 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c100cf0 msgr2=0x7fc04c101140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.847+0000 7fc0523ff700 1 --2- 172.21.15.40:0/540270094 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c100cf0 0x7fc04c101140 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 crypto rx=0x7fc040009990 tx=0x7fc040009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 -- 172.21.15.40:0/540270094 shutdown_connections 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 --2- 172.21.15.40:0/540270094 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c101680 0x7fc04c107f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 --2- 172.21.15.40:0/540270094 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c0ffb00 0x7fc04c0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 --2- 172.21.15.40:0/540270094 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c100cf0 0x7fc04c101140 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 -- 172.21.15.40:0/540270094 >> 172.21.15.40:0/540270094 conn(0x7fc04c0fb0e0 msgr2=0x7fc04c0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:57.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 -- 172.21.15.40:0/540270094 shutdown_connections 2023-07-14T11:11:57.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.848+0000 7fc0523ff700 1 -- 172.21.15.40:0/540270094 wait complete. 2023-07-14T11:11:57.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 Processor -- start 2023-07-14T11:11:57.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 -- start start 2023-07-14T11:11:57.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c100cf0 0x7fc04c197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c101680 0x7fc04c19efc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc04c19f590 con 0x7fc04c101680 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc04c19f6d0 con 0x7fc04c100cf0 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc0523ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc04c19f860 con 0x7fc04c0ffb00 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc04bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.849+0000 7fc04bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44890/0 (socket says 172.21.15.40:44890) 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 -- 172.21.15.40:0/3492542208 learned_addr learned my addr 172.21.15.40:0/3492542208 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc05099c700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c101680 0x7fc04c19efc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04b7fe700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c100cf0 0x7fc04c197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 -- 172.21.15.40:0/3492542208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c100cf0 msgr2=0x7fc04c197f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c100cf0 0x7fc04c197f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 -- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c101680 msgr2=0x7fc04c19efc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:57.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c101680 0x7fc04c19efc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 -- 172.21.15.40:0/3492542208 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0400095f0 con 0x7fc04c0ffb00 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc04bfff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fc03c00e970 tx=0x7fc03c00ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc03c009790 con 0x7fc04c0ffb00 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc03c00c7e0 con 0x7fc04c0ffb00 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.850+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc03c003c70 con 0x7fc04c0ffb00 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.851+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc04c19fb40 con 0x7fc04c0ffb00 2023-07-14T11:11:57.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.851+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc04c1a0050 con 0x7fc04c0ffb00 2023-07-14T11:11:57.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.852+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc04c10b320 con 0x7fc04c0ffb00 2023-07-14T11:11:57.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.854+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fc03c00cc50 con 0x7fc04c0ffb00 2023-07-14T11:11:57.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.855+0000 7fc0497fa700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 0x7fc0340a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:57.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.855+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fc03c014070 con 0x7fc04c0ffb00 2023-07-14T11:11:57.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.855+0000 7fc04b7fe700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 0x7fc0340a36d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:57.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.856+0000 7fc04b7fe700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 0x7fc0340a36d0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fc04000b2e0 tx=0x7fc040005fb0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:57.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:57.857+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc03c096790 con 0x7fc04c0ffb00 2023-07-14T11:11:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:57 smithi130 ceph-mon[129405]: pgmap v347: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:11:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:57 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4080624639' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:57 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2985787946' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:57 smithi040 ceph-mon[97954]: pgmap v347: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:11:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:57 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4080624639' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:57 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2985787946' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:57 smithi045 ceph-mon[128791]: pgmap v347: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:11:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4080624639' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2985787946' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:11:58.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.118+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc04c10b510 con 0x7fc04c0ffb00 2023-07-14T11:11:58.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.119+0000 7fc0497fa700 1 -- 172.21.15.40:0/3492542208 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7fc03c099db0 con 0x7fc04c0ffb00 2023-07-14T11:11:58.122 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:58.122 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:11:58.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 msgr2=0x7fc0340a36d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 0x7fc0340a36d0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fc04000b2e0 tx=0x7fc040005fb0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 msgr2=0x7fc04c197a10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fc03c00e970 tx=0x7fc03c00ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:58.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 shutdown_connections 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fc0340a1220 0x7fc0340a36d0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc04c101680 0x7fc04c19efc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc04c0ffb00 0x7fc04c197a10 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 --2- 172.21.15.40:0/3492542208 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc04c100cf0 0x7fc04c197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 >> 172.21.15.40:0/3492542208 conn(0x7fc04c0fb0e0 msgr2=0x7fc04c106590 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 shutdown_connections 2023-07-14T11:11:58.125 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.121+0000 7fc0523ff700 1 -- 172.21.15.40:0/3492542208 wait complete. 2023-07-14T11:11:58.126 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:11:58.143 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:11:58.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.343+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2663770714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc101680 msgr2=0x7fcbdc107f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.343+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2663770714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc101680 0x7fcbdc107f40 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 crypto rx=0x7fcbd4009f90 tx=0x7fcbd400b330 comp rx=0 tx=0).stop 2023-07-14T11:11:58.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2663770714 shutdown_connections 2023-07-14T11:11:58.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2663770714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc101680 0x7fcbdc107f40 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2663770714 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2663770714 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 0x7fcbdc101140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2663770714 >> 172.21.15.40:0/2663770714 conn(0x7fcbdc0fb0e0 msgr2=0x7fcbdc0fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2663770714 shutdown_connections 2023-07-14T11:11:58.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.344+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2663770714 wait complete. 2023-07-14T11:11:58.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 Processor -- start 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 -- start start 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 0x7fcbdc072900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc101680 0x7fcbdc072e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0735e0 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc073750 con 0x7fcbdc100cf0 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.345+0000 7fcbe47ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc19f930 con 0x7fcbdc101680 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55698/0 (socket says 172.21.15.40:55698) 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 -- 172.21.15.40:0/2778854929 learned_addr learned my addr 172.21.15.40:0/2778854929 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe2d9c700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc101680 0x7fcbdc072e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe1d9a700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 0x7fcbdc072900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 -- 172.21.15.40:0/2778854929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc101680 msgr2=0x7fcbdc072e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc101680 0x7fcbdc072e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 -- 172.21.15.40:0/2778854929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 msgr2=0x7fcbdc072900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 0x7fcbdc072900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 -- 172.21.15.40:0/2778854929 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbd4009c40 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe259b700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7fcbd800e970 tx=0x7fcbd800ec80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd8009790 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.346+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbdc19fc10 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.347+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcbd800c7c0 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.347+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcbd8003c20 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.347+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbdc1a0170 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.348+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbdc106fa0 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.349+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fcbd800c930 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.351+0000 7fcbd37fe700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 0x7fcbc80a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.351+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fcbd8014070 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.351+0000 7fcbe1d9a700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 0x7fcbc80a3650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.352+0000 7fcbe1d9a700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 0x7fcbc80a3650 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc005fd0 tx=0x7fcbcc009310 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.355+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcbd80961f0 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.562+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcbdc02c850 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.564+0000 7fcbd37fe700 1 -- 172.21.15.40:0/2778854929 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v92) v1 ==== 74+0+24481 (secure 0 0 0) 0x7fcbd8099810 con 0x7fcbdc0ffb00 2023-07-14T11:11:58.567 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:11:58.567 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":92,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:11:08.711561+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":5,"flags_names":"hashpspool,ec_overwrites","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"88","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:11:58.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 msgr2=0x7fcbc80a3650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 0x7fcbc80a3650 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc005fd0 tx=0x7fcbcc009310 comp rx=0 tx=0).stop 2023-07-14T11:11:58.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 msgr2=0x7fcbdc0742c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 crypto rx=0x7fcbd800e970 tx=0x7fcbd800ec80 comp rx=0 tx=0).stop 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 shutdown_connections 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fcbc80a11a0 0x7fcbc80a3650 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcbdc101680 0x7fcbdc072e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbdc0ffb00 0x7fcbdc0742c0 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 --2- 172.21.15.40:0/2778854929 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcbdc100cf0 0x7fcbdc072900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 >> 172.21.15.40:0/2778854929 conn(0x7fcbdc0fb0e0 msgr2=0x7fcbdc1040e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 shutdown_connections 2023-07-14T11:11:58.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.566+0000 7fcbe47ff700 1 -- 172.21.15.40:0/2778854929 wait complete. 2023-07-14T11:11:58.588 DEBUG:tasks.fwd_scrub:Yielding 2023-07-14T11:11:58.588 INFO:teuthology.run_tasks:Running task exec... 2023-07-14T11:11:58.598 INFO:teuthology.task.exec:Executing custom commands... 2023-07-14T11:11:58.598 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:11:58.599 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph mgr module enable snap_schedule' 2023-07-14T11:11:58.599 INFO:tasks.fwd_scrub.fs.[cephfs]:start scrubbing fs: cephfs 2023-07-14T11:11:58.599 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2023-07-14T11:11:58.600 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2023-07-14T11:11:58.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 -- 172.21.15.40:0/2495069071 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c072f70 msgr2=0x7f784c0733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 --2- 172.21.15.40:0/2495069071 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c072f70 0x7f784c0733e0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 crypto rx=0x7f7840009990 tx=0x7f7840009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 -- 172.21.15.40:0/2495069071 shutdown_connections 2023-07-14T11:11:58.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 --2- 172.21.15.40:0/2495069071 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c10c0e0 0x7f784c10e4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 --2- 172.21.15.40:0/2495069071 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c074550 0x7f784c0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 --2- 172.21.15.40:0/2495069071 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c072f70 0x7f784c0733e0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 -- 172.21.15.40:0/2495069071 >> 172.21.15.40:0/2495069071 conn(0x7f784c0fb130 msgr2=0x7f784c0fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 -- 172.21.15.40:0/2495069071 shutdown_connections 2023-07-14T11:11:58.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.801+0000 7f7851bff700 1 -- 172.21.15.40:0/2495069071 wait complete. 2023-07-14T11:11:58.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.802+0000 7f7851bff700 1 Processor -- start 2023-07-14T11:11:58.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 -- start start 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c074550 0x7f784c19c490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c10c0e0 0x7f784c19d310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f784c19d970 con 0x7f784c074550 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f784c19dae0 con 0x7f784c072f70 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.803+0000 7f7851bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f784c19dca0 con 0x7f784c10c0e0 2023-07-14T11:11:58.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c10c0e0 0x7f784c19d310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c074550 0x7f784c19c490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49258/0 (socket says 172.21.15.40:49258) 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 -- 172.21.15.40:0/3823823551 learned_addr learned my addr 172.21.15.40:0/3823823551 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 -- 172.21.15.40:0/3823823551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c10c0e0 msgr2=0x7f784c19d310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c10c0e0 0x7f784c19d310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 -- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c074550 msgr2=0x7f784c19c490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c074550 0x7f784c19c490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 -- 172.21.15.40:0/3823823551 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78400095f0 con 0x7f784c072f70 2023-07-14T11:11:58.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f784b7fe700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f783c00d980 tx=0x7f783c00dc90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.804+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f783c00f890 con 0x7f784c072f70 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.805+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f783c010460 con 0x7f784c072f70 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.805+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f783c00e610 con 0x7f784c072f70 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.805+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f784c19df80 con 0x7f784c072f70 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.805+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f784c1a4740 con 0x7f784c072f70 2023-07-14T11:11:58.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.806+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f784c101c30 con 0x7f784c072f70 2023-07-14T11:11:58.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.808+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f783c0105d0 con 0x7f784c072f70 2023-07-14T11:11:58.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.809+0000 7f7848ff9700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 0x7f78340a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.809+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f783c0cc660 con 0x7f784c072f70 2023-07-14T11:11:58.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.809+0000 7f784affd700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 0x7f78340a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.810+0000 7f784affd700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 0x7f78340a3600 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f784000b680 tx=0x7f7840005fd0 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.811+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f783c095a00 con 0x7f784c072f70 2023-07-14T11:11:58.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.814+0000 7fea231ff700 1 -- 172.21.15.40:0/2220686512 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 msgr2=0x7fea1c101120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.814+0000 7fea231ff700 1 --2- 172.21.15.40:0/2220686512 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c101120 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 crypto rx=0x7fea08009990 tx=0x7fea08009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.814+0000 7fea231ff700 1 -- 172.21.15.40:0/2220686512 shutdown_connections 2023-07-14T11:11:58.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 --2- 172.21.15.40:0/2220686512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 0x7fea1c107f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 --2- 172.21.15.40:0/2220686512 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c0ffef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 --2- 172.21.15.40:0/2220686512 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c101120 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 -- 172.21.15.40:0/2220686512 >> 172.21.15.40:0/2220686512 conn(0x7fea1c0fb0e0 msgr2=0x7fea1c0fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 -- 172.21.15.40:0/2220686512 shutdown_connections 2023-07-14T11:11:58.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.815+0000 7fea231ff700 1 -- 172.21.15.40:0/2220686512 wait complete. 2023-07-14T11:11:58.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 Processor -- start 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 -- start start 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c195820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 0x7fea1c19cdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea1c19d3a0 con 0x7fea1c100cd0 2023-07-14T11:11:58.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea1c19d4e0 con 0x7fea1c0ffae0 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea231ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea1c19d670 con 0x7fea1c101660 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea1bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea20f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c195820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea1bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55728/0 (socket says 172.21.15.40:55728) 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.816+0000 7fea1bfff700 1 -- 172.21.15.40:0/176664853 learned_addr learned my addr 172.21.15.40:0/176664853 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea2179c700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 0x7fea1c19cdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 -- 172.21.15.40:0/176664853 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 msgr2=0x7fea1c19cdd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 0x7fea1c19cdd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 -- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 msgr2=0x7fea1c195d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 -- 172.21.15.40:0/176664853 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea080095f0 con 0x7fea1c0ffae0 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea1bfff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea20f9b700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c195820 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fea1000e9c0 tx=0x7fea1000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea19ffb700 1 -- 172.21.15.40:0/176664853 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea10004220 con 0x7fea1c0ffae0 2023-07-14T11:11:58.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea19ffb700 1 -- 172.21.15.40:0/176664853 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea10004380 con 0x7fea1c0ffae0 2023-07-14T11:11:58.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea19ffb700 1 -- 172.21.15.40:0/176664853 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea10003be0 con 0x7fea1c0ffae0 2023-07-14T11:11:58.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.817+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea1c19d950 con 0x7fea1c0ffae0 2023-07-14T11:11:58.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.818+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea1c19de10 con 0x7fea1c0ffae0 2023-07-14T11:11:58.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.821+0000 7fea19ffb700 1 -- 172.21.15.40:0/176664853 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7fea10003d40 con 0x7fea1c0ffae0 2023-07-14T11:11:58.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 Processor -- start 2023-07-14T11:11:58.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 -- start start 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea19ffb700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 0x7fea0c0a3600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96db6760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96db7160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96cbad30 con 0x562f96db6cf0 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96d0dac0 con 0x562f96db5ea0 2023-07-14T11:11:58.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.822+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96d06f50 con 0x562f96db5550 2023-07-14T11:11:58.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea19ffb700 1 -- 172.21.15.40:0/176664853 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fea10014070 con 0x7fea1c0ffae0 2023-07-14T11:11:58.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44948/0 (socket says 172.21.15.40:44948) 2023-07-14T11:11:58.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 -- 172.21.15.40:0/787670498 learned_addr learned my addr 172.21.15.40:0/787670498 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea20f9b700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 0x7fea0c0a3600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96db7160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea1bfff700 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96db6760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 -- 172.21.15.40:0/787670498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 msgr2=0x562f96db6760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96db6760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 -- 172.21.15.40:0/787670498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 msgr2=0x562f96db7160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96db7160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.823+0000 7fea2179c700 1 -- 172.21.15.40:0/787670498 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f96b34e60 con 0x562f96db5550 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.824+0000 7fea2179c700 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fea14008cd0 tx=0x7fea14009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=da80e71e9f8b1c95 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.824+0000 7fea20f9b700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 0x7fea0c0a3600 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fea10099cb0 tx=0x7fea1000cf10 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.824+0000 7fea06ffd700 1 -- 172.21.15.40:0/787670498 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea1401f070 con 0x562f96db5550 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.824+0000 7fea06ffd700 1 -- 172.21.15.40:0/787670498 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea1400f040 con 0x562f96db5550 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.824+0000 7fea06ffd700 1 -- 172.21.15.40:0/787670498 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea1400e050 con 0x562f96db5550 2023-07-14T11:11:58.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 -- 172.21.15.40:0/787670498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 msgr2=0x562f96db5960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fea14008cd0 tx=0x7fea14009920 comp rx=0 tx=0).stop 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 -- 172.21.15.40:0/787670498 shutdown_connections 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96db7160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5550 0x562f96db5960 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/787670498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96db6760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 -- 172.21.15.40:0/787670498 >> 172.21.15.40:0/787670498 conn(0x562f96daae20 msgr2=0x562f96db0250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 -- 172.21.15.40:0/787670498 shutdown_connections 2023-07-14T11:11:58.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.825+0000 7fea36bb7b80 1 -- 172.21.15.40:0/787670498 wait complete. 2023-07-14T11:11:58.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 -- start start 2023-07-14T11:11:58.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 Processor -- start 2023-07-14T11:11:58.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dbf760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc67a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96ca6b40 con 0x562f96db6cf0 2023-07-14T11:11:58.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96ca6840 con 0x562f96db5550 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96b8a9a0 con 0x562f96db5ea0 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.826+0000 7fea36bb7b80 10 client.0 _ll_register_callbacks cb 0x562f96ca47e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49300/0 (socket says 172.21.15.40:49300) 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 -- 172.21.15.40:0/1865645863 learned_addr learned my addr 172.21.15.40:0/1865645863 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc67a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dbf760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 -- 172.21.15.40:0/1865645863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 msgr2=0x562f96dbf760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dbf760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 -- 172.21.15.40:0/1865645863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 msgr2=0x562f96dc67a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc67a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 -- 172.21.15.40:0/1865645863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea140089b0 con 0x562f96db5550 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.827+0000 7fea2179c700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fea140091b0 tx=0x7fea14009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5aaa887df9255205 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fe9fbfff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea1401f070 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562f96c60570 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fea14004ad0 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fea1400e040 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fea36bb7b80 10 client.25535 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x562f96b1dce0 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.828+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x562f96c60570 con 0x562f96db5550 2023-07-14T11:11:58.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fea14012030 con 0x562f96db5550 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 10 client.25535 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 10 client.25535 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 20 client.25535 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96dcad70 0x562f96dcd180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 4 client.25535 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562f96cfff30 con 0x562f96dcad70 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.829+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96dcad70 0x562f96dcd180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.830+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96dcad70 0x562f96dcd180 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=54f16ab797661d45 server_cookie=785a695c8e0e8def in_seq=0 out_seq=0 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.830+0000 7fe9fbfff700 10 client.25535 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.830+0000 7fe9fbfff700 1 -- 172.21.15.40:0/1865645863 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562f96cfff30 con 0x562f96dcad70 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.830+0000 7fe9fbfff700 10 client.25535 handle_command_reply: tid=0 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.830+0000 7fea36bb7b80 1 client.25535 shutdown 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 msgr2=0x562f96dbf220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fea140091b0 tx=0x7fea14009920 comp rx=0 tx=0).stop 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 shutdown_connections 2023-07-14T11:11:58.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96dcad70 0x562f96dcd180 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc67a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fe9fbfff700 0 client.25535 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96dbf220 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1865645863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dbf760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 >> 172.21.15.40:0/1865645863 conn(0x562f96daae20 msgr2=0x562f96daf930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 shutdown_connections 2023-07-14T11:11:58.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1865645863 wait complete. 2023-07-14T11:11:58.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.831+0000 7fea36bb7b80 20 client.25535 trim_cache size 0 max 16384 2023-07-14T11:11:58.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 Processor -- start 2023-07-14T11:11:58.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 -- start start 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dba130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc6730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96db1540 con 0x562f96db6cf0 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96dbe6d0 con 0x562f96db5550 2023-07-14T11:11:58.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.849+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96db6270 con 0x562f96db5ea0 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea2179c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea2179c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49314/0 (socket says 172.21.15.40:49314) 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea2179c700 1 -- 172.21.15.40:0/1764481283 learned_addr learned my addr 172.21.15.40:0/1764481283 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dba130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea2179c700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc6730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 -- 172.21.15.40:0/1764481283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 msgr2=0x562f96db9bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 -- 172.21.15.40:0/1764481283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 msgr2=0x562f96dc6730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc6730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 -- 172.21.15.40:0/1764481283 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f96dbe1a0 con 0x562f96db5ea0 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea2179c700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:58.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.850+0000 7fea1bfff700 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dba130 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x562f96dcda50 tx=0x7fea0800bba0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4e6fb3c2d548166c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea077fe700 1 -- 172.21.15.40:0/1764481283 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea0802b9d0 con 0x562f96db5ea0 2023-07-14T11:11:58.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea077fe700 1 -- 172.21.15.40:0/1764481283 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea08025040 con 0x562f96db5ea0 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea077fe700 1 -- 172.21.15.40:0/1764481283 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea0801fb60 con 0x562f96db5ea0 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1764481283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 msgr2=0x562f96dba130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dba130 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x562f96dcda50 tx=0x7fea0800bba0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1764481283 shutdown_connections 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db6cf0 0x562f96dc6730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5550 0x562f96db9bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/1764481283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db5ea0 0x562f96dba130 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1764481283 >> 172.21.15.40:0/1764481283 conn(0x562f96daae20 msgr2=0x562f96dbd0c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1764481283 shutdown_connections 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.851+0000 7fea36bb7b80 1 -- 172.21.15.40:0/1764481283 wait complete. 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 -- start start 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 Processor -- start 2023-07-14T11:11:58.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db6cf0 0x562f96dc5160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96dc56a0 con 0x562f96db5550 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96dc5810 con 0x562f96db5ea0 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.852+0000 7fea36bb7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f96dc59d0 con 0x562f96db6cf0 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea36bb7b80 10 client.0 _ll_register_callbacks cb 0x7fea14008bd0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:58.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea2179c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49324/0 (socket says 172.21.15.40:49324) 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea2179c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55756/0 (socket says 172.21.15.40:55756) 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 -- 172.21.15.40:0/3275881977 learned_addr learned my addr 172.21.15.40:0/3275881977 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea2179c700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db6cf0 0x562f96dc5160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 -- 172.21.15.40:0/3275881977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db6cf0 msgr2=0x562f96dc5160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db6cf0 0x562f96dc5160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 -- 172.21.15.40:0/3275881977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 msgr2=0x562f96dc3c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea1bfff700 1 -- 172.21.15.40:0/3275881977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562f96b1dce0 con 0x562f96db5ea0 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.853+0000 7fea2179c700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fea1bfff700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fea08009520 tx=0x7fea08005940 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7a24d684a86dd6fb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fe9fbfff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fea0802bbd0 con 0x562f96db5ea0 2023-07-14T11:11:58.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7fea0802bd30 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fea1400f040 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fea0801f8e0 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fea36bb7b80 10 client.25541 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7fea1c108850 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.854+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fea1400f040 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fea0803e070 con 0x562f96db5ea0 2023-07-14T11:11:58.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 10 client.25541 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 10 client.25541 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 20 client.25541 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96e20d40 0x562f96e23150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 4 client.25541 mds_command: new command op to 24569 tid=0[{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x562f96cc20d0 con 0x562f96e20d40 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea2179c700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96e20d40 0x562f96e23150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fea2179c700 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96e20d40 0x562f96e23150 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=725fb3900891276 server_cookie=9148a852ad253fea in_seq=0 out_seq=0 2023-07-14T11:11:58.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.855+0000 7fe9fbfff700 10 client.25541 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:58.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fe9fbfff700 1 -- 172.21.15.40:0/3275881977 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x562f96cc20d0 con 0x562f96e20d40 2023-07-14T11:11:58.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fe9fbfff700 10 client.25541 handle_command_reply: tid=0 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 client.25541 shutdown 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 msgr2=0x562f96dc4150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fea08009520 tx=0x7fea08005940 comp rx=0 tx=0).stop 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 shutdown_connections 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562f96e20d40 0x562f96e23150 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f96db6cf0 0x562f96dc5160 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f96db5550 0x562f96dc3c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fe9fbfff700 0 client.25541 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 --2- 172.21.15.40:0/3275881977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f96db5ea0 0x562f96dc4150 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.856+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 >> 172.21.15.40:0/3275881977 conn(0x562f96daae20 msgr2=0x562f96db9bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 shutdown_connections 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea36bb7b80 1 -- 172.21.15.40:0/3275881977 wait complete. 2023-07-14T11:11:58.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea36bb7b80 20 client.25541 trim_cache size 0 max 16384 2023-07-14T11:11:58.869 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:58.869 INFO:teuthology.orchestra.run.smithi040.stdout: "return_code": 0, 2023-07-14T11:11:58.869 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_tag": "f61327bc-e73c-4a5f-a553-cfa4db97459a", 2023-07-14T11:11:58.869 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": "asynchronous" 2023-07-14T11:11:58.869 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:58.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 msgr2=0x7fea0c0a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 0x7fea0c0a3600 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7fea10099cb0 tx=0x7fea1000cf10 comp rx=0 tx=0).stop 2023-07-14T11:11:58.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 msgr2=0x7fea1c195820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:58.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c195820 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7fea1000e9c0 tx=0x7fea1000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 shutdown_connections 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7fea0c0a1150 0x7fea0c0a3600 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fea1c101660 0x7fea1c19cdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fea1c0ffae0 0x7fea1c195820 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 --2- 172.21.15.40:0/176664853 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea1c100cd0 0x7fea1c195d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.857+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 >> 172.21.15.40:0/176664853 conn(0x7fea1c0fb0e0 msgr2=0x7fea1c0fd350 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.858+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 shutdown_connections 2023-07-14T11:11:58.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:58.858+0000 7fea231ff700 1 -- 172.21.15.40:0/176664853 wait complete. 2023-07-14T11:11:58.875 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3492542208' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:58.876 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2778854929' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:58.878 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:11:59.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3492542208' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2778854929' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3492542208' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:11:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2778854929' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:11:59.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 -- 172.21.15.40:0/496819280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4074550 msgr2=0x7f02e40729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/496819280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4074550 0x7f02e40729a0 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 crypto rx=0x7f02e0009990 tx=0x7f02e0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 -- 172.21.15.40:0/496819280 shutdown_connections 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/496819280 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 0x7f02e410e490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/496819280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4074550 0x7f02e40729a0 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/496819280 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4072f70 0x7f02e40733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 -- 172.21.15.40:0/496819280 >> 172.21.15.40:0/496819280 conn(0x7f02e40fb0e0 msgr2=0x7f02e40fd530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 -- 172.21.15.40:0/496819280 shutdown_connections 2023-07-14T11:11:59.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.091+0000 7f02eb5ff700 1 -- 172.21.15.40:0/496819280 wait complete. 2023-07-14T11:11:59.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 Processor -- start 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 -- start start 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 0x7f02e419c420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 0x7f02e41a3490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e41a3a60 con 0x7f02e4072f70 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e41a3ba0 con 0x7f02e4074550 2023-07-14T11:11:59.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.092+0000 7f02eb5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e41a3d30 con 0x7f02e410c0b0 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55776/0 (socket says 172.21.15.40:55776) 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 -- 172.21.15.40:0/3443430968 learned_addr learned my addr 172.21.15.40:0/3443430968 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 0x7f02e41a3490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 0x7f02e419c420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 -- 172.21.15.40:0/3443430968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 msgr2=0x7f02e41a3490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 0x7f02e41a3490 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 -- 172.21.15.40:0/3443430968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 msgr2=0x7f02e419c420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 0x7f02e419c420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 -- 172.21.15.40:0/3443430968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02e00095f0 con 0x7f02e4072f70 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 0x7f02e419c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.093+0000 7f02e939b700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f02e00098e0 tx=0x7f02e0009580 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.094+0000 7f02da7fc700 1 -- 172.21.15.40:0/3443430968 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02e001d070 con 0x7f02e4072f70 2023-07-14T11:11:59.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.094+0000 7f02da7fc700 1 -- 172.21.15.40:0/3443430968 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f02e000be40 con 0x7f02e4072f70 2023-07-14T11:11:59.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.094+0000 7f02da7fc700 1 -- 172.21.15.40:0/3443430968 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02e0017650 con 0x7f02e4072f70 2023-07-14T11:11:59.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.094+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02e41a3fb0 con 0x7f02e4072f70 2023-07-14T11:11:59.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.094+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02e41a4320 con 0x7f02e4072f70 2023-07-14T11:11:59.100 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.098+0000 7f02da7fc700 1 -- 172.21.15.40:0/3443430968 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 135715+0+0 (secure 0 0 0) 0x7f02e0021410 con 0x7f02e4072f70 2023-07-14T11:11:59.101 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.098+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "snap_schedule"} v 0) v1 -- 0x7f784c02cb00 con 0x7f784c072f70 2023-07-14T11:11:59.101 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.098+0000 7f02fefdab80 1 Processor -- start 2023-07-14T11:11:59.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 -- start start 2023-07-14T11:11:59.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 0x56430625d8e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625de20 0x56430625e6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x564306162cb0 con 0x56430625d4d0 2023-07-14T11:11:59.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643061b5a40 con 0x56430625de20 2023-07-14T11:11:59.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02fefdab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643061aeed0 con 0x56430625ec70 2023-07-14T11:11:59.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e9b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 0x56430625d8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e939b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625de20 0x56430625e6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44968/0 (socket says 172.21.15.40:44968) 2023-07-14T11:11:59.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e9b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 0x56430625d8e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55790/0 (socket says 172.21.15.40:55790) 2023-07-14T11:11:59.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.099+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3985713444 learned_addr learned my addr 172.21.15.40:0/3985713444 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:59.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02da7fc700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 0x7f02d00a3680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3985713444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625de20 msgr2=0x56430625e6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625de20 0x56430625e6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3985713444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 msgr2=0x56430625d8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 0x56430625d8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3985713444 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564305fdcde0 con 0x56430625ec70 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02da7fc700 1 -- 172.21.15.40:0/3443430968 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f02e00ce0b0 con 0x7f02e4072f70 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 0x7f02d00a3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f02d400bd40 tx=0x7f02d400f640 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d364bff2295bbb47 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.100+0000 7f02cf7fe700 1 -- 172.21.15.40:0/3985713444 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d400fda0 con 0x56430625ec70 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02cf7fe700 1 -- 172.21.15.40:0/3985713444 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f02d400be90 con 0x56430625ec70 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02cf7fe700 1 -- 172.21.15.40:0/3985713444 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d401b790 con 0x56430625ec70 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 -- 172.21.15.40:0/3985713444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 msgr2=0x56430625f0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 0x7f02d00a3680 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f02dc009ec0 tx=0x7f02dc009040 comp rx=0 tx=0).ready entity=mgr.14148 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f02d400bd40 tx=0x7f02d400f640 comp rx=0 tx=0).stop 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 -- 172.21.15.40:0/3985713444 shutdown_connections 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x56430625f0e0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625d4d0 0x56430625d8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3985713444 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625de20 0x56430625e6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.101+0000 7f02fefdab80 1 -- 172.21.15.40:0/3985713444 >> 172.21.15.40:0/3985713444 conn(0x564306252da0 msgr2=0x5643062581d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.102+0000 7f02fefdab80 1 -- 172.21.15.40:0/3985713444 shutdown_connections 2023-07-14T11:11:59.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.102+0000 7f02fefdab80 1 -- 172.21.15.40:0/3985713444 wait complete. 2023-07-14T11:11:59.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.102+0000 7f02fefdab80 1 -- start start 2023-07-14T11:11:59.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.102+0000 7f02fefdab80 1 Processor -- start 2023-07-14T11:11:59.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x564306267110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626e690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430614eac0 con 0x56430625de20 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430614e7c0 con 0x56430625ec70 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x564306032920 con 0x56430625d4d0 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02fefdab80 10 client.0 _ll_register_callbacks cb 0x56430614c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55796/0 (socket says 172.21.15.40:55796) 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3691603810 learned_addr learned my addr 172.21.15.40:0/3691603810 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:59.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e939b700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x564306267110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3691603810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 msgr2=0x564306267110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x564306267110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 msgr2=0x56430626e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626e690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e8b9a700 1 -- 172.21.15.40:0/3691603810 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02d400b9a0 con 0x56430625de20 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.103+0000 7f02e939b700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x564306267110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f02d4000c00 tx=0x7f02d400fdd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e441aafa157d1689 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02ccff9700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d4020070 con 0x56430625de20 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5643061084f0 con 0x56430625de20 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f02d4011650 con 0x56430625de20 2023-07-14T11:11:59.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f02d4011ce0 con 0x56430625de20 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02fefdab80 10 client.15558 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.104+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x564305fc5c60 con 0x56430625de20 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5643061084f0 con 0x56430625de20 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f02d40243f0 con 0x56430625de20 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 10 client.15558 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 10 client.15558 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:11:59.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 20 client.15558 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564306272e40 0x564306275250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 4 client.15558 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5643061a7eb0 con 0x564306272e40 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.105+0000 7f02e939b700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564306272e40 0x564306275250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.106+0000 7f02e939b700 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564306272e40 0x564306275250 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e121195a838e5cb9 server_cookie=faf1713016ba5d10 in_seq=0 out_seq=0 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.106+0000 7f02ccff9700 10 client.15558 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:59.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.106+0000 7f02ccff9700 1 -- 172.21.15.40:0/3691603810 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5643061a7eb0 con 0x564306272e40 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.106+0000 7f02ccff9700 10 client.15558 handle_command_reply: tid=0 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.106+0000 7f02fefdab80 1 client.15558 shutdown 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 msgr2=0x564306267650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 crypto rx=0x7f02d4000c00 tx=0x7f02d400fdd0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 shutdown_connections 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564306272e40 0x564306275250 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.119 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x564306267110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3691603810 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x564306267650 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02ccff9700 0 client.15558 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 >> 172.21.15.40:0/3691603810 conn(0x564306252da0 msgr2=0x5643062578b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 shutdown_connections 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 1 -- 172.21.15.40:0/3691603810 wait complete. 2023-07-14T11:11:59.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.107+0000 7f02fefdab80 20 client.15558 trim_cache size 0 max 16384 2023-07-14T11:11:59.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.124+0000 7f02fefdab80 1 Processor -- start 2023-07-14T11:11:59.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 -- start start 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625d4d0 0x564306265080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x564306261bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x564306266c80 con 0x56430625de20 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430624fef0 con 0x56430625d4d0 2023-07-14T11:11:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02fefdab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5643062592e0 con 0x56430625ec70 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55798/0 (socket says 172.21.15.40:55798) 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02e8b9a700 1 -- 172.21.15.40:0/2634145644 learned_addr learned my addr 172.21.15.40:0/2634145644 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02e939b700 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625d4d0 0x564306265080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.125+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x564306261bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 -- 172.21.15.40:0/2634145644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 msgr2=0x564306261bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x564306261bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 -- 172.21.15.40:0/2634145644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625d4d0 msgr2=0x564306265080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625d4d0 0x564306265080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 -- 172.21.15.40:0/2634145644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564306003250 con 0x56430625de20 2023-07-14T11:11:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f02d4005770 tx=0x7f02d400b880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e4e2c135b26df6e1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02cffff700 1 -- 172.21.15.40:0/2634145644 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d4020030 con 0x56430625de20 2023-07-14T11:11:59.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02cffff700 1 -- 172.21.15.40:0/2634145644 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f02d400b5e0 con 0x56430625de20 2023-07-14T11:11:59.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.126+0000 7f02cffff700 1 -- 172.21.15.40:0/2634145644 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d400dc90 con 0x56430625de20 2023-07-14T11:11:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- 172.21.15.40:0/2634145644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 msgr2=0x56430626e650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 crypto rx=0x7f02d4005770 tx=0x7f02d400b880 comp rx=0 tx=0).stop 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- 172.21.15.40:0/2634145644 shutdown_connections 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625ec70 0x564306261bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625d4d0 0x564306265080 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 --2- 172.21.15.40:0/2634145644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626e650 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- 172.21.15.40:0/2634145644 >> 172.21.15.40:0/2634145644 conn(0x564306252da0 msgr2=0x564306265e80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- 172.21.15.40:0/2634145644 shutdown_connections 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- 172.21.15.40:0/2634145644 wait complete. 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.127+0000 7f02fefdab80 1 -- start start 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 Processor -- start 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626adc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626dbf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430626e130 con 0x56430625de20 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430626e2a0 con 0x56430625ec70 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56430626e460 con 0x56430625d4d0 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02fefdab80 10 client.0 _ll_register_callbacks cb 0x7f02e0097370 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02e939b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02e8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626adc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02e939b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44986/0 (socket says 172.21.15.40:44986) 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02e939b700 1 -- 172.21.15.40:0/3718592886 learned_addr learned my addr 172.21.15.40:0/3718592886 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.128+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626dbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 -- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 msgr2=0x56430626dbf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626dbf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 -- 172.21.15.40:0/3718592886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 msgr2=0x56430626adc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626adc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 -- 172.21.15.40:0/3718592886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564305fc5c60 con 0x56430625d4d0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e9b9c700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02e939b700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f02e00cbbd0 tx=0x7f02e0009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6e603920a50dde5b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02ccff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02e0005c00 con 0x56430625d4d0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f02d400be90 con 0x56430625d4d0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f02e009b200 con 0x56430625d4d0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f02e009f020 con 0x56430625d4d0 2023-07-14T11:11:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02fefdab80 10 client.25564 fetch_fsmap learned FSMap version 27 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.129+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x564305fdb2b0 con 0x56430625d4d0 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f02d400be90 con 0x56430625d4d0 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f02e0097060 con 0x56430625d4d0 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 10 client.25564 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 10 client.25564 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 20 client.25564 populate_metadata read hostname 'smithi040' 2023-07-14T11:11:59.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5643062c7c00 0x5643062ca010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 4 client.25564 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56430616a050 con 0x5643062c7c00 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.130+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5643062c7c00 0x5643062ca010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02e8b9a700 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5643062c7c00 0x5643062ca010 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=58efb4e7dede64e3 server_cookie=59f06bbb06a137e8 in_seq=0 out_seq=0 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02ccff9700 10 client.25564 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02ccff9700 1 -- 172.21.15.40:0/3718592886 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x56430616a050 con 0x5643062c7c00 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02ccff9700 10 client.25564 handle_command_reply: tid=0 2023-07-14T11:11:59.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 client.25564 shutdown 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 msgr2=0x56430626a880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f02e00cbbd0 tx=0x7f02e0009310 comp rx=0 tx=0).stop 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 shutdown_connections 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5643062c7c00 0x5643062ca010 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56430625ec70 0x56430626dbf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02ccff9700 0 client.25564 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56430625d4d0 0x56430626a880 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 --2- 172.21.15.40:0/3718592886 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56430625de20 0x56430626adc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 >> 172.21.15.40:0/3718592886 conn(0x564306252da0 msgr2=0x564306265080 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 shutdown_connections 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.131+0000 7f02fefdab80 1 -- 172.21.15.40:0/3718592886 wait complete. 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02fefdab80 20 client.25564 trim_cache size 0 max 16384 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:11:59.144 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "IDLE (waiting for more scrubs)", 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: "f61327bc-e73c-4a5f-a553-cfa4db97459a": { 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "f61327bc-e73c-4a5f-a553-cfa4db97459a", 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:11:59.145 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:11:59.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 msgr2=0x7f02d00a3680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 0x7f02d00a3680 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f02dc009ec0 tx=0x7f02dc009040 comp rx=0 tx=0).stop 2023-07-14T11:11:59.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 msgr2=0x7f02e419bee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 crypto rx=0x7f02e00098e0 tx=0x7f02e0009580 comp rx=0 tx=0).stop 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 shutdown_connections 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f02d00a11d0 0x7f02d00a3680 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e410c0b0 0x7f02e41a3490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e4072f70 0x7f02e419bee0 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 --2- 172.21.15.40:0/3443430968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e4074550 0x7f02e419c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 >> 172.21.15.40:0/3443430968 conn(0x7f02e40fb0e0 msgr2=0x7f02e40fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 shutdown_connections 2023-07-14T11:11:59.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.132+0000 7f02eb5ff700 1 -- 172.21.15.40:0/3443430968 wait complete. 2023-07-14T11:11:59.152 INFO:tasks.cephfs.filesystem:scrub status for tag:f61327bc-e73c-4a5f-a553-cfa4db97459a - {'path': '/', 'tag': 'f61327bc-e73c-4a5f-a553-cfa4db97459a', 'options': 'recursive,force'} 2023-07-14T11:11:59.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.759+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mgrmap(e 17) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f783c0108b0 con 0x7f784c072f70 2023-07-14T11:11:59.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.760+0000 7f7848ff9700 1 -- 172.21.15.40:0/3823823551 <== mon.2 v2:172.21.15.45:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "snap_schedule"}]=0 v17) v1 ==== 92+0+0 (secure 0 0 0) 0x7f783c08df30 con 0x7f784c072f70 2023-07-14T11:11:59.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.763+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 msgr2=0x7f78340a3600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.763+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 0x7f78340a3600 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f784000b680 tx=0x7f7840005fd0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 msgr2=0x7f784c19bf50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:11:59.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f783c00d980 tx=0x7f783c00dc90 comp rx=0 tx=0).stop 2023-07-14T11:11:59.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 shutdown_connections 2023-07-14T11:11:59.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f78340a1150 0x7f78340a3600 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f784c10c0e0 0x7f784c19d310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f784c072f70 0x7f784c19bf50 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 --2- 172.21.15.40:0/3823823551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f784c074550 0x7f784c19c490 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:11:59.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 >> 172.21.15.40:0/3823823551 conn(0x7f784c0fb130 msgr2=0x7f784c0fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:11:59.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 shutdown_connections 2023-07-14T11:11:59.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:11:59.764+0000 7f7851bff700 1 -- 172.21.15.40:0/3823823551 wait complete. 2023-07-14T11:11:59.788 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/allow_m_granularity true' 2023-07-14T11:12:00.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 -- 172.21.15.40:0/486843742 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 msgr2=0x7f45dc109e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 --2- 172.21.15.40:0/486843742 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc109e50 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f45d400b0a0 tx=0x7f45d400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 -- 172.21.15.40:0/486843742 shutdown_connections 2023-07-14T11:12:00.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 --2- 172.21.15.40:0/486843742 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc109e50 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 --2- 172.21.15.40:0/486843742 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc0729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 --2- 172.21.15.40:0/486843742 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 0x7f45dc0733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 -- 172.21.15.40:0/486843742 >> 172.21.15.40:0/486843742 conn(0x7f45dc0fb130 msgr2=0x7f45dc0fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:00.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 -- 172.21.15.40:0/486843742 shutdown_connections 2023-07-14T11:12:00.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.001+0000 7f45e21ff700 1 -- 172.21.15.40:0/486843742 wait complete. 2023-07-14T11:12:00.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.002+0000 7f45e21ff700 1 Processor -- start 2023-07-14T11:12:00.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.002+0000 7f45e21ff700 1 -- start start 2023-07-14T11:12:00.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 0x7f45dc197ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc198f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45dc199660 con 0x7f45dc072f70 2023-07-14T11:12:00.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45dc1997d0 con 0x7f45dc1079d0 2023-07-14T11:12:00.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45e21ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f45dc199990 con 0x7f45dc074550 2023-07-14T11:12:00.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45db7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 0x7f45dc197ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:45008/0 (socket says 172.21.15.40:45008) 2023-07-14T11:12:00.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45daffd700 1 -- 172.21.15.40:0/1524018568 learned_addr learned my addr 172.21.15.40:0/1524018568 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:00.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.003+0000 7f45dbfff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc198f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 msgr2=0x7f45dc198f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc198f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 msgr2=0x7f45dc197ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 0x7f45dc197ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 -- 172.21.15.40:0/1524018568 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f45cc009520 con 0x7f45dc074550 2023-07-14T11:12:00.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45daffd700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f45d000e970 tx=0x7f45d000ec80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:00.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45d0009790 con 0x7f45dc074550 2023-07-14T11:12:00.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f45d4009d00 con 0x7f45dc074550 2023-07-14T11:12:00.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f45d000c7e0 con 0x7f45dc074550 2023-07-14T11:12:00.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f45d0003c70 con 0x7f45dc074550 2023-07-14T11:12:00.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.004+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f45dc1a0390 con 0x7f45dc074550 2023-07-14T11:12:00.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.005+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f45dc106080 con 0x7f45dc074550 2023-07-14T11:12:00.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.007+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 17) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f45d000cc50 con 0x7f45dc074550 2023-07-14T11:12:00.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.009+0000 7f45d8ff9700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 0x7f45c40a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.009+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f45d0014070 con 0x7f45dc074550 2023-07-14T11:12:00.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.009+0000 7f45db7fe700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 msgr2=0x7f45c40a3650 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:00.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.009+0000 7f45db7fe700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 0x7f45c40a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:12:00.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.014+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f45d00963b0 con 0x7f45dc074550 2023-07-14T11:12:00.019 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: pgmap v348: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:12:00.019 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: scrub queued for path: / 2023-07-14T11:12:00.019 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.020 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:12:00.020 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.020 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3823823551' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.020 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.021 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: ignoring --setuser ceph since I am not root 2023-07-14T11:12:00.021 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: ignoring --setgroup ceph since I am not root 2023-07-14T11:12:00.021 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-mgr[131271]: -- 172.21.15.130:0/2386399767 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55baa2ef11e0 con 0x55baa2ec6c00 2023-07-14T11:12:00.022 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:11:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:11:59.932+0000 7f02cf6aa200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:12:00.022 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.016+0000 7f02cf6aa200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:00.040 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setuser ceph since I am not root 2023-07-14T11:12:00.041 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: ignoring --setgroup ceph since I am not root 2023-07-14T11:12:00.041 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: pgmap v348: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:12:00.041 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: scrub queued for path: / 2023-07-14T11:12:00.041 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.041 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:12:00.042 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.042 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3823823551' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.042 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:11:59 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.058 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: ignoring --setuser ceph since I am not root 2023-07-14T11:12:00.058 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: ignoring --setgroup ceph since I am not root 2023-07-14T11:12:00.058 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mgr[130266]: -- 172.21.15.45:0/3625398759 <== mon.2 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56138f8171e0 con 0x56138f7ecc00 2023-07-14T11:12:00.058 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:11:59.958+0000 7f82c8a79200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:00.056+0000 7f82c8a79200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: pgmap v348: 129 pgs: 129 active+clean; 587 KiB data, 330 MiB used, 1.0 TiB / 1.0 TiB avail; 4.7 KiB/s rd, 170 B/s wr, 9 op/s 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: scrub queued for path: / 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:00.059 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3823823551' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.060 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:11:59 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "snap_schedule"}]: dispatch 2023-07-14T11:12:00.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.210+0000 7f45db7fe700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 msgr2=0x7f45c40a3650 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:00.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.210+0000 7f45db7fe700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 0x7f45c40a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:12:00.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.223+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}] v 0) v1 -- 0x7f45dc04e2f0 con 0x7f45dc074550 2023-07-14T11:12:00.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.233+0000 7f45d8ff9700 1 -- 172.21.15.40:0/1524018568 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/allow_m_granularity}]=0 v23)=0 v23) v1 ==== 138+0+0 (secure 0 0 0) 0x7f45d0003dd0 con 0x7f45dc074550 2023-07-14T11:12:00.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 msgr2=0x7f45c40a3650 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:12:00.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 0x7f45c40a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 msgr2=0x7f45dc198220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f45d000e970 tx=0x7f45d000ec80 comp rx=0 tx=0).stop 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 shutdown_connections 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f45c40a11a0 0x7f45c40a3650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f45dc1079d0 0x7f45dc198f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f45dc072f70 0x7f45dc197ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 --2- 172.21.15.40:0/1524018568 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f45dc074550 0x7f45dc198220 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.235+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 >> 172.21.15.40:0/1524018568 conn(0x7f45dc0fb130 msgr2=0x7f45dc109c70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.236+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 shutdown_connections 2023-07-14T11:12:00.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.236+0000 7f45e21ff700 1 -- 172.21.15.40:0/1524018568 wait complete. 2023-07-14T11:12:00.257 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph config set mgr mgr/snap_schedule/dump_on_update true' 2023-07-14T11:12:00.292 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.107+0000 7f02cf6aa200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:12:00.292 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.199+0000 7f02cf6aa200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:12:00.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:00.038+0000 7fc79a7a0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-07-14T11:12:00.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:00.146+0000 7fc79a7a0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:00.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:00.267+0000 7fc79a7a0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:12:00.313 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:00.166+0000 7f82c8a79200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-07-14T11:12:00.314 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:00.279+0000 7f82c8a79200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:12:00.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 -- 172.21.15.40:0/4051295817 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 msgr2=0x7f3b140733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 --2- 172.21.15.40:0/4051295817 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b140733e0 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7f3b08009990 tx=0x7f3b08009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 -- 172.21.15.40:0/4051295817 shutdown_connections 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 --2- 172.21.15.40:0/4051295817 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b1410c0d0 0x7f3b1410e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 --2- 172.21.15.40:0/4051295817 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b14074550 0x7f3b140729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 --2- 172.21.15.40:0/4051295817 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b140733e0 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 -- 172.21.15.40:0/4051295817 >> 172.21.15.40:0/4051295817 conn(0x7f3b140fb0e0 msgr2=0x7f3b140fd550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 -- 172.21.15.40:0/4051295817 shutdown_connections 2023-07-14T11:12:00.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.459+0000 7f3b18dff700 1 -- 172.21.15.40:0/4051295817 wait complete. 2023-07-14T11:12:00.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.460+0000 7f3b18dff700 1 Processor -- start 2023-07-14T11:12:00.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.460+0000 7f3b18dff700 1 -- start start 2023-07-14T11:12:00.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b14074550 0x7f3b1419c4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 0x7f3b1419d320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b1419d980 con 0x7f3b1410c0d0 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b1419daf0 con 0x7f3b14072f70 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b18dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b1419dcb0 con 0x7f3b14074550 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49410/0 (socket says 172.21.15.40:49410) 2023-07-14T11:12:00.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 -- 172.21.15.40:0/2194707283 learned_addr learned my addr 172.21.15.40:0/2194707283 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b11d9b700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b14074550 0x7f3b1419c4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b12d9d700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 0x7f3b1419d320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b14074550 msgr2=0x7f3b1419c4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b14074550 0x7f3b1419c4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 msgr2=0x7f3b1419d320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 0x7f3b1419d320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b1259c700 1 -- 172.21.15.40:0/2194707283 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b080095f0 con 0x7f3b14072f70 2023-07-14T11:12:00.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.461+0000 7f3b12d9d700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 0x7f3b1419d320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b1259c700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f3b0400e970 tx=0x7f3b0400ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b04009790 con 0x7f3b14072f70 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b0400eec0 con 0x7f3b14072f70 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b04005590 con 0x7f3b14072f70 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b1419df90 con 0x7f3b14072f70 2023-07-14T11:12:00.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.462+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b141a4750 con 0x7f3b14072f70 2023-07-14T11:12:00.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.463+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b14101c20 con 0x7f3b14072f70 2023-07-14T11:12:00.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.465+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 17) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f3b0400cbd0 con 0x7f3b14072f70 2023-07-14T11:12:00.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.467+0000 7f3b037fe700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 0x7f3afc0a3650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.467+0000 7f3b11d9b700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 msgr2=0x7f3afc0a3650 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:00.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.467+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f3b04014070 con 0x7f3b14072f70 2023-07-14T11:12:00.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.467+0000 7f3b11d9b700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 0x7f3afc0a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:12:00.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.473+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3b040969a0 con 0x7f3b14072f70 2023-07-14T11:12:00.630 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:00.395+0000 7fc79a7a0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-07-14T11:12:00.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.668+0000 7f3b11d9b700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 msgr2=0x7f3afc0a3650 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:00.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.668+0000 7f3b11d9b700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 0x7f3afc0a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:12:00.683 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.372+0000 7f02cf6aa200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:12:00.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.685+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command([{prefix=config set, name=mgr/snap_schedule/dump_on_update}] v 0) v1 -- 0x7f3b1402cb00 con 0x7f3b14072f70 2023-07-14T11:12:00.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.693+0000 7f3b037fe700 1 -- 172.21.15.40:0/2194707283 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/snap_schedule/dump_on_update}]=0 v24)=0 v24) v1 ==== 133+0+0 (secure 0 0 0) 0x7f3b04083e60 con 0x7f3b14072f70 2023-07-14T11:12:00.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 msgr2=0x7f3afc0a3650 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 0x7f3afc0a3650 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 msgr2=0x7f3b1419bf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7f3b0400e970 tx=0x7f3b0400ec80 comp rx=0 tx=0).stop 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 shutdown_connections 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f3afc0a11a0 0x7f3afc0a3650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b1410c0d0 0x7f3b1419d320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b14072f70 0x7f3b1419bf60 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 --2- 172.21.15.40:0/2194707283 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b14074550 0x7f3b1419c4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 >> 172.21.15.40:0/2194707283 conn(0x7f3b140fb0e0 msgr2=0x7f3b140fd4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:00.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 shutdown_connections 2023-07-14T11:12:00.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.696+0000 7f3b18dff700 1 -- 172.21.15.40:0/2194707283 wait complete. 2023-07-14T11:12:00.719 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule add --fs=cephfs --path=/ --snap_schedule=1M' 2023-07-14T11:12:00.768 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:00.492+0000 7f82c8a79200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:12:00.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 -- 172.21.15.40:0/3151509033 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78102f40 msgr2=0x7f8d78105320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/3151509033 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78102f40 0x7f8d78105320 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7f8d6c009990 tx=0x7f8d6c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 -- 172.21.15.40:0/3151509033 shutdown_connections 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/3151509033 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d78105860 0x7f8d78107c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/3151509033 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d78100620 0x7f8d78102a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/3151509033 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78102f40 0x7f8d78105320 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 -- 172.21.15.40:0/3151509033 >> 172.21.15.40:0/3151509033 conn(0x7f8d780fa000 msgr2=0x7f8d780fc450 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:00.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 -- 172.21.15.40:0/3151509033 shutdown_connections 2023-07-14T11:12:00.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.962+0000 7f8d7dfff700 1 -- 172.21.15.40:0/3151509033 wait complete. 2023-07-14T11:12:00.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.963+0000 7f8d7dfff700 1 Processor -- start 2023-07-14T11:12:00.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.963+0000 7f8d7dfff700 1 -- start start 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.963+0000 7f8d7dfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 0x7f8d78197bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d7dfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d78102f40 0x7f8d781980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d7dfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d78105860 0x7f8d78198f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d7dfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d781995d0 con 0x7f8d78102f40 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d7dfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d78199740 con 0x7f8d78100620 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d7dfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d78199900 con 0x7f8d78105860 2023-07-14T11:12:00.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 0x7f8d78197bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 0x7f8d78197bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49424/0 (socket says 172.21.15.40:49424) 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 -- 172.21.15.40:0/506119015 learned_addr learned my addr 172.21.15.40:0/506119015 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d78102f40 0x7f8d781980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d77fff700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d78105860 0x7f8d78198f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d78105860 msgr2=0x7f8d78198f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d78105860 0x7f8d78198f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d78102f40 msgr2=0x7f8d781980f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:00.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d78102f40 0x7f8d781980f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d6c0095f0 con 0x7f8d78100620 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.964+0000 7f8d777fe700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 0x7f8d78197bb0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f8d6800e970 tx=0x7f8d6800ed30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.965+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d68009790 con 0x7f8d78100620 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.965+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8d68004cf0 con 0x7f8d78100620 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.965+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8d78199be0 con 0x7f8d78100620 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.965+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d68005680 con 0x7f8d78100620 2023-07-14T11:12:00.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.965+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d781a03a0 con 0x7f8d78100620 2023-07-14T11:12:00.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.966+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8d781073d0 con 0x7f8d78100620 2023-07-14T11:12:00.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.970+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 17) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f8d68004750 con 0x7f8d78100620 2023-07-14T11:12:00.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.971+0000 7f8d74ff9700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:00.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.971+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:00.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.971+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(92..92 src has 1..92) v4 ==== 9086+0+0 (secure 0 0 0) 0x7f8d68014070 con 0x7f8d78100620 2023-07-14T11:12:00.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.971+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2023-07-14T11:12:00.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:00.976+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8d680969e0 con 0x7f8d78100620 2023-07-14T11:12:01.000 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.681+0000 7f02cf6aa200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:12:01.001 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.773+0000 7f02cf6aa200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:12:01.001 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.902+0000 7f02cf6aa200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:12:01.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2023-07-14T11:12:01.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-mon[129405]: mgrmap e17: x(active, since 8m), standbys: z, y 2023-07-14T11:12:01.003 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.003 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2023-07-14T11:12:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-mon[97954]: mgrmap e17: x(active, since 8m), standbys: z, y 2023-07-14T11:12:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:00.627+0000 7fc79a7a0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-07-14T11:12:01.064 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:00.891+0000 7f82c8a79200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:12:01.064 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:01.004+0000 7f82c8a79200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:12:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "snap_schedule"}]': finished 2023-07-14T11:12:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-mon[128791]: mgrmap e17: x(active, since 8m), standbys: z, y 2023-07-14T11:12:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:00 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' 2023-07-14T11:12:01.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:01.172+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:01.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:01.172+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2023-07-14T11:12:01.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:01.214+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] -- mgr_command(tid 0: {"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1M", "target": ["mon-mgr", ""]}) v1 -- 0x7f8d78060950 con 0x7f8d600a1220 2023-07-14T11:12:01.292 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:00.998+0000 7f02cf6aa200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:12:01.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:01.042+0000 7fc79a7a0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-07-14T11:12:01.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:01.162+0000 7fc79a7a0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-07-14T11:12:01.563 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:01.156+0000 7f82c8a79200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:12:01.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:01.271+0000 7f82c8a79200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:12:01.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:01.573+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:01.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:01.573+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2023-07-14T11:12:01.792 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:01.418+0000 7f02cf6aa200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:12:01.795 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:01.330+0000 7fc79a7a0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-07-14T11:12:01.795 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:01.458+0000 7fc79a7a0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-07-14T11:12:02.063 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:01.783+0000 7f82c8a79200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:12:02.292 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:01.980+0000 7f02cf6aa200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:12:02.293 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:02.150+0000 7f02cf6aa200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:12:02.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:02.015+0000 7fc79a7a0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-07-14T11:12:02.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:02.374+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:02.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:02.374+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2023-07-14T11:12:02.813 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:02.468+0000 7f82c8a79200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:12:02.814 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:02.661+0000 7f82c8a79200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:12:03.045 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:02.783+0000 7fc79a7a0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-07-14T11:12:03.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:02.998+0000 7fc79a7a0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-07-14T11:12:03.177 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:02.913+0000 7f02cf6aa200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:03.178 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:03.004+0000 7f02cf6aa200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:12:03.454 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:03.175+0000 7f02cf6aa200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:03.792 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:03.452+0000 7f02cf6aa200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:12:03.793 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:03.680+0000 7f02cf6aa200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:12:03.957 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:03.625+0000 7f82c8a79200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:03.958 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:03.737+0000 7f82c8a79200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:12:03.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:03.975+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:03.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:03.975+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2023-07-14T11:12:04.090 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:03.828+0000 7f02cf6aa200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:12:04.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:03.979+0000 7fc79a7a0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:04.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:04.103+0000 7fc79a7a0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-07-14T11:12:04.304 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:03.955+0000 7f82c8a79200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:04.542 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:04.088+0000 7f02cf6aa200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:12:04.557 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:04.302+0000 7f82c8a79200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:12:04.706 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:04.331+0000 7fc79a7a0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-07-14T11:12:04.813 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:04.555+0000 7f82c8a79200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:12:04.814 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:04.737+0000 7f82c8a79200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:12:04.989 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:04.704+0000 7fc79a7a0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-07-14T11:12:05.113 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:04.790+0000 7f02cf6aa200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:12:05.113 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:04.921+0000 7f02cf6aa200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:12:05.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:04.986+0000 7fc79a7a0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-07-14T11:12:05.296 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:05.184+0000 7fc79a7a0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-07-14T11:12:05.442 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:05.110+0000 7f02cf6aa200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:12:05.442 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:05.223+0000 7f02cf6aa200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:05.563 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:05.065+0000 7f82c8a79200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:12:05.792 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:05.439+0000 7f02cf6aa200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:12:05.795 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:05.540+0000 7fc79a7a0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-07-14T11:12:06.289 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:05.890+0000 7f82c8a79200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:12:06.289 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:06.047+0000 7f82c8a79200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:12:06.372 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:06.370+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mgrmap(e 18) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f8d68004a00 con 0x7f8d78100620 2023-07-14T11:12:06.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:06.287+0000 7f82c8a79200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:12:06.564 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:06.429+0000 7f82c8a79200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:06.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: Standby manager daemon z restarted 2023-07-14T11:12:06.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: Standby manager daemon z started 2023-07-14T11:12:06.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:12:06.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:06.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:12:06.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:06.688 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:06.503+0000 7fc79a7a0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: Standby manager daemon z restarted 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: Standby manager daemon z started 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:12:06.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:06.792 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z[131247]: 2023-07-14T11:12:06.294+0000 7f02cf6aa200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:12:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: Standby manager daemon z restarted 2023-07-14T11:12:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: Standby manager daemon z started 2023-07-14T11:12:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/crt"}]: dispatch 2023-07-14T11:12:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/z/key"}]: dispatch 2023-07-14T11:12:06.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:06 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.130:0/3261015519' entity='mgr.z' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:06.945 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:06.685+0000 7fc79a7a0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-07-14T11:12:07.063 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:06.699+0000 7f82c8a79200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:12:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:07.178+0000 7f8d76ffd700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/2087222828 2023-07-14T11:12:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:07.178+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 6.400000 2023-07-14T11:12:07.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:06.942+0000 7fc79a7a0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-07-14T11:12:07.295 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:07.097+0000 7fc79a7a0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-07-14T11:12:07.741 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:07 smithi045 ceph-mon[128791]: mgrmap e18: x(active, since 8m), standbys: y, z 2023-07-14T11:12:07.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:07 smithi130 ceph-mon[129405]: mgrmap e18: x(active, since 8m), standbys: y, z 2023-07-14T11:12:07.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:07 smithi040 ceph-mon[97954]: mgrmap e18: x(active, since 8m), standbys: y, z 2023-07-14T11:12:07.795 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:07.383+0000 7fc79a7a0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-07-14T11:12:08.063 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:12:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y[130242]: 2023-07-14T11:12:07.739+0000 7f82c8a79200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:12:08.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:08.392+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 8 ==== mgrmap(e 19) v1 ==== 135732+0+0 (secure 0 0 0) 0x7f8d680cade0 con 0x7f8d78100620 2023-07-14T11:12:08.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:08.518+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 9 ==== mgrmap(e 20) v1 ==== 135400+0+0 (secure 0 0 0) 0x7f8d68096db0 con 0x7f8d78100620 2023-07-14T11:12:08.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:08.518+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 msgr2=0x7f8d600a36d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-07-14T11:12:08.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:08.518+0000 7f8d74ff9700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/2087222828,v1:172.21.15.40:6801/2087222828] conn(0x7f8d600a1220 0x7f8d600a36d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: Standby manager daemon y restarted 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: Standby manager daemon y started 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:12:08.678 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:08 smithi045 ceph-mon[128791]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:08.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: Standby manager daemon y restarted 2023-07-14T11:12:08.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: Standby manager daemon y started 2023-07-14T11:12:08.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:12:08.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:08.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:12:08.680 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:08 smithi130 ceph-mon[129405]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:08.687 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.498+0000 7fc79a7a0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-07-14T11:12:08.687 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: Standby manager daemon y restarted 2023-07-14T11:12:08.687 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: Standby manager daemon y started 2023-07-14T11:12:08.687 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-07-14T11:12:08.687 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-07-14T11:12:08.688 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-07-14T11:12:08.688 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-mon[97954]: from='mgr.? 172.21.15.45:0/4012482947' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.721+0000 7fc6f9cbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.721+0000 7fc6f9cbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.721+0000 7fc6f9cbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.721+0000 7fc6f9cbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.721+0000 7fc6f9cbf700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.046 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.722+0000 7fc6f64b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.722+0000 7fc6f64b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.722+0000 7fc6f64b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.722+0000 7fc6f64b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:12:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:12:08.722+0000 7fc6f64b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:12:09.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.522+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mon.2 v2:172.21.15.45:3300/0 10 ==== mgrmap(e 21) v1 ==== 135527+0+0 (secure 0 0 0) 0x7f8d68097680 con 0x7f8d78100620 2023-07-14T11:12:09.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.522+0000 7f8d74ff9700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d60065660 0x7f8d60065a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:09.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.522+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule add", "fs": "cephfs", "path": "/", "snap_schedule": "1M", "target": ["mon-mgr", ""]}) v1 -- 0x7f8d78060950 con 0x7f8d60065660 2023-07-14T11:12:09.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.523+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d60065660 0x7f8d60065a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:09.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.526+0000 7f8d76ffd700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d60065660 0x7f8d60065a30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8d6c00b2e0 tx=0x7f8d6c009510 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:09.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.595+0000 7f8d74ff9700 1 -- 172.21.15.40:0/506119015 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f8d78060950 con 0x7f8d60065660 2023-07-14T11:12:09.598 INFO:teuthology.orchestra.run.smithi040.stdout:Schedule set for path / 2023-07-14T11:12:09.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.597+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d60065660 msgr2=0x7f8d60065a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:09.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.597+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d60065660 0x7f8d60065a30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8d6c00b2e0 tx=0x7f8d6c009510 comp rx=0 tx=0).stop 2023-07-14T11:12:09.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.597+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 msgr2=0x7f8d78197bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:09.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.597+0000 7f8d7dfff700 1 --2- 172.21.15.40:0/506119015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d78100620 0x7f8d78197bb0 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7f8d6800e970 tx=0x7f8d6800ed30 comp rx=0 tx=0).stop 2023-07-14T11:12:09.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.598+0000 7f8d77fff700 1 -- 172.21.15.40:0/506119015 reap_dead start 2023-07-14T11:12:09.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.598+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 shutdown_connections 2023-07-14T11:12:09.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.598+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 >> 172.21.15.40:0/506119015 conn(0x7f8d780fa000 msgr2=0x7f8d78101280 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:09.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.598+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 shutdown_connections 2023-07-14T11:12:09.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.598+0000 7f8d7dfff700 1 -- 172.21.15.40:0/506119015 wait complete. 2023-07-14T11:12:09.621 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule retention add --fs=cephfs --path=/ --retention-spec-or-period=6M3h' 2023-07-14T11:12:09.689 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: mgrmap e19: x(active, since 9m), standbys: z, y 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: Active manager daemon x restarted 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: Activating manager daemon x 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: osdmap e93: 12 total, 12 up, 12 in 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: mgrmap e20: x(active, starting, since 0.0116662s), standbys: z, y 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:12:09.690 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:12:09.691 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:12:09.692 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:12:09.693 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:12:09.694 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:12:09.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: Manager daemon x is now available 2023-07-14T11:12:09.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:12:09.695 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:09 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: mgrmap e19: x(active, since 9m), standbys: z, y 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: Active manager daemon x restarted 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: Activating manager daemon x 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: osdmap e93: 12 total, 12 up, 12 in 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: mgrmap e20: x(active, starting, since 0.0116662s), standbys: z, y 2023-07-14T11:12:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:12:09.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:12:09.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:12:09.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:12:09.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: Manager daemon x is now available 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:12:09.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:09 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:12:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: mgrmap e19: x(active, since 9m), standbys: z, y 2023-07-14T11:12:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: Active manager daemon x restarted 2023-07-14T11:12:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: Activating manager daemon x 2023-07-14T11:12:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: osdmap e93: 12 total, 12 up, 12 in 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: mgrmap e20: x(active, starting, since 0.0116662s), standbys: z, y 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "j"}]: dispatch 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "k"}]: dispatch 2023-07-14T11:12:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:12:09.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "z", "id": "z"}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-07-14T11:12:09.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 8}]: dispatch 2023-07-14T11:12:09.818 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 9}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 10}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 11}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: Manager daemon x is now available 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.819 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:09.820 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-07-14T11:12:09.820 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:09 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-07-14T11:12:09.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 -- 172.21.15.40:0/3577280507 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 msgr2=0x7fb2d80733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:09.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/3577280507 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d80733e0 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fb2cc009990 tx=0x7fb2cc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 -- 172.21.15.40:0/3577280507 shutdown_connections 2023-07-14T11:12:09.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/3577280507 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d81079d0 0x7fb2d8109e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/3577280507 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d8074550 0x7fb2d80729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/3577280507 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d80733e0 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 -- 172.21.15.40:0/3577280507 >> 172.21.15.40:0/3577280507 conn(0x7fb2d80fb130 msgr2=0x7fb2d80fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:09.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.838+0000 7fb2de7ff700 1 -- 172.21.15.40:0/3577280507 shutdown_connections 2023-07-14T11:12:09.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.839+0000 7fb2de7ff700 1 -- 172.21.15.40:0/3577280507 wait complete. 2023-07-14T11:12:09.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.839+0000 7fb2de7ff700 1 Processor -- start 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 -- start start 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d8197b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 0x7fb2d81980c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 0x7fb2d8198f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2d81995a0 con 0x7fb2d81079d0 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2d8199710 con 0x7fb2d8072f70 2023-07-14T11:12:09.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2de7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2d81998d0 con 0x7fb2d8074550 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d8197b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2d77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 0x7fb2d81980c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d8197b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46390/0 (socket says 172.21.15.40:46390) 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.840+0000 7fb2d7fff700 1 -- 172.21.15.40:0/1588422753 learned_addr learned my addr 172.21.15.40:0/1588422753 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2dcd9c700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 0x7fb2d8198f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 -- 172.21.15.40:0/1588422753 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 msgr2=0x7fb2d8197b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d8197b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 -- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 msgr2=0x7fb2d8198f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:09.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 0x7fb2d8198f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:09.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 -- 172.21.15.40:0/1588422753 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2cc0095f0 con 0x7fb2d8074550 2023-07-14T11:12:09.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2dcd9c700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 0x7fb2d8198f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:09.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d77fe700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 0x7fb2d81980c0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb2cc00b680 tx=0x7fb2cc009580 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2cc01d070 con 0x7fb2d8074550 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2d8199b50 con 0x7fb2d8074550 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.841+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fb2cc021410 con 0x7fb2d8074550 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.842+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb2cc017650 con 0x7fb2d8074550 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.842+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2d80ff6a0 con 0x7fb2d8074550 2023-07-14T11:12:09.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.843+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb2d8106080 con 0x7fb2d8074550 2023-07-14T11:12:09.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.845+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 21) v1 ==== 135527+0+0 (secure 0 0 0) 0x7fb2cc0217a0 con 0x7fb2d8074550 2023-07-14T11:12:09.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.847+0000 7fb2d57fa700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 0x7fb2c00a3480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:09.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.847+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fb2cc0ce4b0 con 0x7fb2d8074550 2023-07-14T11:12:09.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.847+0000 7fb2d7fff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 0x7fb2c00a3480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:09.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.848+0000 7fb2d7fff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 0x7fb2c00a3480 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb2c8005950 tx=0x7fb2c8018040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:09.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:09.848+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fb2cc097b30 con 0x7fb2d8074550 2023-07-14T11:12:10.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.092+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6M3h", "target": ["mon-mgr", ""]}) v1 -- 0x7fb2d8060950 con 0x7fb2c00a0fd0 2023-07-14T11:12:10.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.146+0000 7fb2d57fa700 1 -- 172.21.15.40:0/1588422753 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+26 (secure 0 0 0) 0x7fb2d8060950 con 0x7fb2c00a0fd0 2023-07-14T11:12:10.149 INFO:teuthology.orchestra.run.smithi040.stdout:Retention added to path / 2023-07-14T11:12:10.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 msgr2=0x7fb2c00a3480 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 0x7fb2c00a3480 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fb2c8005950 tx=0x7fb2c8018040 comp rx=0 tx=0).stop 2023-07-14T11:12:10.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 msgr2=0x7fb2d81980c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 0x7fb2d81980c0 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb2cc00b680 tx=0x7fb2cc009580 comp rx=0 tx=0).stop 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 shutdown_connections 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fb2c00a0fd0 0x7fb2c00a3480 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2d81079d0 0x7fb2d8198f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fb2d8072f70 0x7fb2d8197b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.148+0000 7fb2de7ff700 1 --2- 172.21.15.40:0/1588422753 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fb2d8074550 0x7fb2d81980c0 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.149+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 >> 172.21.15.40:0/1588422753 conn(0x7fb2d80fb130 msgr2=0x7fb2d80fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:10.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.149+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 shutdown_connections 2023-07-14T11:12:10.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.149+0000 7fb2de7ff700 1 -- 172.21.15.40:0/1588422753 wait complete. 2023-07-14T11:12:10.170 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/2416934307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 msgr2=0x7f3f480733e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/2416934307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f480733e0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 crypto rx=0x7f3f38009990 tx=0x7f3f38009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/2416934307 shutdown_connections 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/2416934307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 0x7f3f4810e4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/2416934307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f480729a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/2416934307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f480733e0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/2416934307 >> 172.21.15.40:0/2416934307 conn(0x7f3f480fb130 msgr2=0x7f3f480fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:10.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/2416934307 shutdown_connections 2023-07-14T11:12:10.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.382+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/2416934307 wait complete. 2023-07-14T11:12:10.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 Processor -- start 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 -- start start 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f4819c4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 0x7f3f4819d340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.383+0000 7f3f4f7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819d9a0 con 0x7f3f48072f70 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4f7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819db10 con 0x7f3f48074550 2023-07-14T11:12:10.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4f7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819dcd0 con 0x7f3f4810c0e0 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:36384/0 (socket says 172.21.15.40:36384) 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 -- 172.21.15.40:0/1498300385 learned_addr learned my addr 172.21.15.40:0/1498300385 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4cd9a700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f4819c4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4dd9c700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 0x7f3f4819d340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 -- 172.21.15.40:0/1498300385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 msgr2=0x7f3f4819d340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 0x7f3f4819d340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 -- 172.21.15.40:0/1498300385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 msgr2=0x7f3f4819c4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f4819c4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4d59b700 1 -- 172.21.15.40:0/1498300385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f44009520 con 0x7f3f48072f70 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.384+0000 7f3f4cd9a700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f4819c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f4d59b700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f3f4400ea30 tx=0x7f3f4400ed40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f4400cbd0 con 0x7f3f48072f70 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f380095f0 con 0x7f3f48072f70 2023-07-14T11:12:10.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f4400eed0 con 0x7f3f48072f70 2023-07-14T11:12:10.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f44003d20 con 0x7f3f48072f70 2023-07-14T11:12:10.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.385+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f481a4590 con 0x7f3f48072f70 2023-07-14T11:12:10.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.386+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f48101c30 con 0x7f3f48072f70 2023-07-14T11:12:10.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.388+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 21) v1 ==== 135527+0+0 (secure 0 0 0) 0x7f3f44003ee0 con 0x7f3f48072f70 2023-07-14T11:12:10.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.389+0000 7f3f3e7fc700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 0x7f3f340a3400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.390+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3f44014070 con 0x7f3f48072f70 2023-07-14T11:12:10.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.390+0000 7f3f4cd9a700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 0x7f3f340a3400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.390+0000 7f3f4cd9a700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 0x7f3f340a3400 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f3f3800b2e0 tx=0x7f3f38005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:10.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.395+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3f44096280 con 0x7f3f48072f70 2023-07-14T11:12:10.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.530+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f3f44096650 con 0x7f3f48072f70 2023-07-14T11:12:10.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.650+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) v1 -- 0x7f3f48060950 con 0x7f3f340a0f50 2023-07-14T11:12:10.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.659+0000 7f3f3e7fc700 1 -- 172.21.15.40:0/1498300385 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+283 (secure 0 0 0) 0x7f3f48060950 con 0x7f3f340a0f50 2023-07-14T11:12:10.661 INFO:teuthology.orchestra.run.smithi040.stdout:{"fs": "cephfs", "subvol": null, "path": "/", "rel_path": "/", "schedule": "1M", "retention": {"h": 3, "M": 6}, "start": "2023-07-14T00:00:00", "created": "2023-07-14T11:12:09", "first": null, "last": null, "last_pruned": null, "created_count": 0, "pruned_count": 0, "active": true} 2023-07-14T11:12:10.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 msgr2=0x7f3f340a3400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 0x7f3f340a3400 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f3f3800b2e0 tx=0x7f3f38005fb0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 msgr2=0x7f3f4819bf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 crypto rx=0x7f3f4400ea30 tx=0x7f3f4400ed40 comp rx=0 tx=0).stop 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 shutdown_connections 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f340a0f50 0x7f3f340a3400 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f4810c0e0 0x7f3f4819d340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48072f70 0x7f3f4819bf80 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 --2- 172.21.15.40:0/1498300385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48074550 0x7f3f4819c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 >> 172.21.15.40:0/1498300385 conn(0x7f3f480fb130 msgr2=0x7f3f480fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:10.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 shutdown_connections 2023-07-14T11:12:10.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.661+0000 7f3f4f7ff700 1 -- 172.21.15.40:0/1498300385 wait complete. 2023-07-14T11:12:10.683 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2023-07-14T11:12:10.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:10 smithi130 ceph-mon[129405]: mgrmap e21: x(active, since 1.01758s), standbys: z, y 2023-07-14T11:12:10.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:10 smithi130 ceph-mon[129405]: [14/Jul/2023:11:12:09] ENGINE Bus STARTING 2023-07-14T11:12:10.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:10 smithi040 ceph-mon[97954]: mgrmap e21: x(active, since 1.01758s), standbys: z, y 2023-07-14T11:12:10.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:10 smithi040 ceph-mon[97954]: [14/Jul/2023:11:12:09] ENGINE Bus STARTING 2023-07-14T11:12:10.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:10 smithi045 ceph-mon[128791]: mgrmap e21: x(active, since 1.01758s), standbys: z, y 2023-07-14T11:12:10.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:10 smithi045 ceph-mon[128791]: [14/Jul/2023:11:12:09] ENGINE Bus STARTING 2023-07-14T11:12:10.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 -- 172.21.15.40:0/4069761493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8074550 msgr2=0x7fcfc80729a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 --2- 172.21.15.40:0/4069761493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8074550 0x7fcfc80729a0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fcfb8009990 tx=0x7fcfb8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 -- 172.21.15.40:0/4069761493 shutdown_connections 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 --2- 172.21.15.40:0/4069761493 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc81079d0 0x7fcfc8109e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 --2- 172.21.15.40:0/4069761493 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8074550 0x7fcfc80729a0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 --2- 172.21.15.40:0/4069761493 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc8072f70 0x7fcfc80733e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 -- 172.21.15.40:0/4069761493 >> 172.21.15.40:0/4069761493 conn(0x7fcfc80fb130 msgr2=0x7fcfc80fd560 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:10.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 -- 172.21.15.40:0/4069761493 shutdown_connections 2023-07-14T11:12:10.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.892+0000 7fcfccdff700 1 -- 172.21.15.40:0/4069761493 wait complete. 2023-07-14T11:12:10.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.893+0000 7fcfccdff700 1 Processor -- start 2023-07-14T11:12:10.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.893+0000 7fcfccdff700 1 -- start start 2023-07-14T11:12:10.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8072f70 0x7fcfc8197bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc81079d0 0x7fcfc8198f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfc81995d0 con 0x7fcfc81079d0 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfc8199740 con 0x7fcfc8074550 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfccdff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfc8199900 con 0x7fcfc8072f70 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46430/0 (socket says 172.21.15.40:46430) 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 -- 172.21.15.40:0/3632674007 learned_addr learned my addr 172.21.15.40:0/3632674007 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:10.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc6d9d700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc81079d0 0x7fcfc8198f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc659c700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8072f70 0x7fcfc8197bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 -- 172.21.15.40:0/3632674007 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8072f70 msgr2=0x7fcfc8197bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8072f70 0x7fcfc8197bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 -- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc81079d0 msgr2=0x7fcfc8198f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc81079d0 0x7fcfc8198f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.894+0000 7fcfc5d9b700 1 -- 172.21.15.40:0/3632674007 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfc0009c40 con 0x7fcfc8074550 2023-07-14T11:12:10.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfc5d9b700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fcfbc00e970 tx=0x7fcfbc00ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:10.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfbc009790 con 0x7fcfc8074550 2023-07-14T11:12:10.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcfbc00c7c0 con 0x7fcfc8074550 2023-07-14T11:12:10.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcfbc003c20 con 0x7fcfc8074550 2023-07-14T11:12:10.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcfb80095f0 con 0x7fcfc8074550 2023-07-14T11:12:10.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.895+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcfc81a01c0 con 0x7fcfc8074550 2023-07-14T11:12:10.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.896+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcfc8106080 con 0x7fcfc8074550 2023-07-14T11:12:10.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.898+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7fcfbc00c930 con 0x7fcfc8074550 2023-07-14T11:12:10.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.900+0000 7fcfb77fe700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 0x7fcfb00a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:10.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.900+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fcfbc014070 con 0x7fcfc8074550 2023-07-14T11:12:10.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.900+0000 7fcfc659c700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 0x7fcfb00a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:10.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.901+0000 7fcfc659c700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 0x7fcfb00a3880 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fcfb800b2e0 tx=0x7fcfb8005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:10.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:10.901+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcfbc0969f0 con 0x7fcfc8074550 2023-07-14T11:12:11.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.146+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) v1 -- 0x7fcfc8060950 con 0x7fcfb00a13d0 2023-07-14T11:12:11.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.154+0000 7fcfb77fe700 1 -- 172.21.15.40:0/3632674007 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+10 (secure 0 0 0) 0x7fcfc8060950 con 0x7fcfb00a13d0 2023-07-14T11:12:11.157 INFO:teuthology.orchestra.run.smithi040.stdout:/ 1M 6M3h 2023-07-14T11:12:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.156+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 msgr2=0x7fcfb00a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.156+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 0x7fcfb00a3880 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fcfb800b2e0 tx=0x7fcfb8005fb0 comp rx=0 tx=0).stop 2023-07-14T11:12:11.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.156+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 msgr2=0x7fcfc81980f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.156+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7fcfbc00e970 tx=0x7fcfbc00ec80 comp rx=0 tx=0).stop 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 shutdown_connections 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcfb00a13d0 0x7fcfb00a3880 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcfc81079d0 0x7fcfc8198f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcfc8072f70 0x7fcfc8197bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 --2- 172.21.15.40:0/3632674007 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcfc8074550 0x7fcfc81980f0 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:11.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 >> 172.21.15.40:0/3632674007 conn(0x7fcfc80fb130 msgr2=0x7fcfc80fd510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 shutdown_connections 2023-07-14T11:12:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:11.157+0000 7fcfccdff700 1 -- 172.21.15.40:0/3632674007 wait complete. 2023-07-14T11:12:11.178 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > START_TIME' 2023-07-14T11:12:11.250 INFO:teuthology.run_tasks:Running task full_sequential_finally... 2023-07-14T11:12:11.261 INFO:teuthology.run_tasks:Running task exec... 2023-07-14T11:12:11.276 INFO:teuthology.task.exec:Executing custom commands... 2023-07-14T11:12:11.276 INFO:teuthology.task.exec:Running commands on role client.0 host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:12:11.276 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -l /tmp/cephfs/ && sudo df -h /tmp/cephfs/' 2023-07-14T11:12:11.385 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:11.431 INFO:teuthology.orchestra.run.smithi040.stdout:Filesystem Size Used Avail Use% Mounted on 2023-07-14T11:12:11.432 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:3300,172.21.15.45:3300,172.21.15.130:3300:/ 1.1T 332M 1.1T 1% /tmp/cephfs 2023-07-14T11:12:11.437 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo mkdir -m 755 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres && sudo chown postgres:postgres /tmp/cephfs/postgres' 2023-07-14T11:12:11.586 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- mkdir -m 700 --context=system_u:system_r:postgresql_t:s0 /tmp/cephfs/postgres/data' 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: [14/Jul/2023:11:12:09] ENGINE Serving on http://172.21.15.40:8765 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: [14/Jul/2023:11:12:10] ENGINE Serving on https://172.21.15.40:7150 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: [14/Jul/2023:11:12:10] ENGINE Bus STARTED 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6M3h", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: pgmap v4: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:11.683 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: mgrmap e22: x(active, since 2s), standbys: z, y 2023-07-14T11:12:11.684 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:11 smithi040 ceph-mon[97954]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:11.699 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo semanage fcontext -a -t postgresql_db_t "/tmp/cephfs/postgres(/.*)?"' 2023-07-14T11:12:11.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: [14/Jul/2023:11:12:09] ENGINE Serving on http://172.21.15.40:8765 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: [14/Jul/2023:11:12:10] ENGINE Serving on https://172.21.15.40:7150 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: [14/Jul/2023:11:12:10] ENGINE Bus STARTED 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6M3h", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: pgmap v4: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: mgrmap e22: x(active, since 2s), standbys: z, y 2023-07-14T11:12:11.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:11 smithi130 ceph-mon[129405]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:11.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: [14/Jul/2023:11:12:09] ENGINE Serving on http://172.21.15.40:8765 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: [14/Jul/2023:11:12:10] ENGINE Serving on https://172.21.15.40:7150 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: [14/Jul/2023:11:12:10] ENGINE Bus STARTED 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: from='client.25639 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule retention add", "fs": "cephfs", "path": "/", "retention_spec_or_period": "6M3h", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: pgmap v4: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: mgrmap e22: x(active, since 2s), standbys: z, y 2023-07-14T11:12:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:11 smithi045 ceph-mon[128791]: from='client.15648 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:13.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:12 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:12 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='client.25652 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:13.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:13.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:12 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: pgmap v5: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:12:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:12:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:12:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:12:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:13 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:12:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: pgmap v5: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:12:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:12:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:12:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:12:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:13 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:12:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: pgmap v5: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:12:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:12:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:12:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:12:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:12:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:13 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:12:15.185 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:12:15.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi040:/etc/ceph/ceph.conf 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: pgmap v6: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s rd, 1 op/s 2023-07-14T11:12:15.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:14 smithi040 ceph-mon[97954]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.190 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo grep -i postgresql /etc/selinux/targeted/contexts/files/file_contexts.local' 2023-07-14T11:12:15.292 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi040:/etc/ceph/ceph.conf 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:12:15.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: pgmap v6: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s rd, 1 op/s 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:14 smithi130 ceph-mon[129405]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.296 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres(/.*)? system_u:object_r:postgresql_db_t:s0 2023-07-14T11:12:15.301 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo restorecon -R -v /tmp/cephfs/postgres' 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi040:/etc/ceph/ceph.conf 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi045:/etc/ceph/ceph.conf 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi130:/etc/ceph/ceph.conf 2023-07-14T11:12:15.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.conf 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: pgmap v6: 129 pgs: 129 active+clean; 973 KiB data, 331 MiB used, 1.0 TiB / 1.0 TiB avail; 0 B/s rd, 1 op/s 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:14 smithi045 ceph-mon[128791]: Updating smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:15.418 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -lZaR /tmp/cephfs/postgres/' 2023-07-14T11:12:15.520 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/: 2023-07-14T11:12:15.520 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:15.520 INFO:teuthology.orchestra.run.smithi040.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:15.520 INFO:teuthology.orchestra.run.smithi040.stdout:drwxrwxrwt. 3 root root system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 .. 2023-07-14T11:12:15.520 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 data 2023-07-14T11:12:15.521 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:15.521 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data: 2023-07-14T11:12:15.521 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:15.521 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:15.521 INFO:teuthology.orchestra.run.smithi040.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 .. 2023-07-14T11:12:15.525 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo mkdir -p /etc/systemd/system/postgresql.service.d/ && printf '"'"'[Service]\nEnvironment=PGDATA=/tmp/cephfs/postgres/data\nEnvironment=PGLOG=/tmp/cephfs/postgres/pgstartup.log\n'"'"' | sudo tee /etc/systemd/system/postgresql.service.d/env.conf' 2023-07-14T11:12:15.657 INFO:teuthology.orchestra.run.smithi040.stdout:[Service] 2023-07-14T11:12:15.657 INFO:teuthology.orchestra.run.smithi040.stdout:Environment=PGDATA=/tmp/cephfs/postgres/data 2023-07-14T11:12:15.657 INFO:teuthology.orchestra.run.smithi040.stdout:Environment=PGLOG=/tmp/cephfs/postgres/pgstartup.log 2023-07-14T11:12:15.662 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- postgresql-setup --initdb' 2023-07-14T11:12:15.869 INFO:teuthology.orchestra.run.smithi040.stderr: * Initializing database in '/tmp/cephfs/postgres/data' 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: Updating smithi040:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:16.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:16 smithi130 ceph-mon[129405]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:16.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.295 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: Updating smithi040:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:16.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:16 smithi040 ceph-mon[97954]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:16.313 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:16.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: Updating smithi040:/etc/ceph/ceph.client.admin.keyring 2023-07-14T11:12:16.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:16 smithi045 ceph-mon[128791]: Updating smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/config/ceph.client.admin.keyring 2023-07-14T11:12:17.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:17.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:17 smithi130 ceph-mon[129405]: pgmap v7: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 132 KiB/s wr, 54 op/s 2023-07-14T11:12:17.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.545 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:17 smithi040 ceph-mon[97954]: pgmap v7: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 132 KiB/s wr, 54 op/s 2023-07-14T11:12:17.563 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:12:17.564 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:17 smithi045 ceph-mon[128791]: pgmap v7: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 132 KiB/s wr, 54 op/s 2023-07-14T11:12:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:19 smithi130 ceph-mon[129405]: pgmap v8: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 103 KiB/s wr, 42 op/s 2023-07-14T11:12:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:19 smithi040 ceph-mon[97954]: pgmap v8: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 103 KiB/s wr, 42 op/s 2023-07-14T11:12:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:19 smithi045 ceph-mon[128791]: pgmap v8: 129 pgs: 129 active+clean; 1.2 MiB data, 332 MiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 103 KiB/s wr, 42 op/s 2023-07-14T11:12:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:21 smithi130 ceph-mon[129405]: pgmap v9: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 197 KiB/s rd, 2.0 MiB/s wr, 63 op/s 2023-07-14T11:12:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:21 smithi040 ceph-mon[97954]: pgmap v9: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 197 KiB/s rd, 2.0 MiB/s wr, 63 op/s 2023-07-14T11:12:22.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:21 smithi045 ceph-mon[128791]: pgmap v9: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 197 KiB/s rd, 2.0 MiB/s wr, 63 op/s 2023-07-14T11:12:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:23 smithi130 ceph-mon[129405]: pgmap v10: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 1.9 MiB/s wr, 58 op/s 2023-07-14T11:12:24.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:23 smithi040 ceph-mon[97954]: pgmap v10: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 1.9 MiB/s wr, 58 op/s 2023-07-14T11:12:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:23 smithi045 ceph-mon[128791]: pgmap v10: 129 pgs: 129 active+clean; 23 MiB data, 376 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 1.9 MiB/s wr, 58 op/s 2023-07-14T11:12:24.540 INFO:teuthology.orchestra.run.smithi040.stderr: * Initialized, logs are in /var/lib/pgsql/initdb_postgresql.log 2023-07-14T11:12:24.546 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ls -lZaR /tmp/cephfs/postgres/' 2023-07-14T11:12:24.663 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/: 2023-07-14T11:12:24.663 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.663 INFO:teuthology.orchestra.run.smithi040.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:drwxrwxrwt. 3 root root system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 .. 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 data 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data: 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:total 30 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 . 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:drwxr-xr-x. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 .. 2023-07-14T11:12:24.664 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 base 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 58 Jul 14 11:12 global 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 log 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_commit_ts 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_dynshmem 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 4269 Jul 14 11:12 pg_hba.conf 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 1636 Jul 14 11:12 pg_ident.conf 2023-07-14T11:12:24.665 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 pg_logical 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 pg_multixact 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 pg_notify 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_replslot 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_serial 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_snapshots 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_stat 2023-07-14T11:12:24.666 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_stat_tmp 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 pg_subtrans 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_tblspc 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 pg_twophase 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 PG_VERSION 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 pg_wal 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 pg_xact 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 88 Jul 14 11:12 postgresql.auto.conf 2023-07-14T11:12:24.667 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 22989 Jul 14 11:12 postgresql.conf 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/base: 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 . 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 1 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 12346 2023-07-14T11:12:24.668 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 12347 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/base/1: 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:total 7217 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 . 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 .. 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 112 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 113 2023-07-14T11:12:24.669 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Jul 14 11:12 12182 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12182_fsm 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12182_vm 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12184 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12186 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12187_fsm 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187_vm 2023-07-14T11:12:24.670 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12189 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12191 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12192_fsm 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192_vm 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12194 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12196 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197 2023-07-14T11:12:24.671 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12197_fsm 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197_vm 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12199 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12201 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12202_fsm 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202_vm 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12204 2023-07-14T11:12:24.672 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12206 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12207_fsm 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207_vm 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12209 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12211 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12212 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12214 2023-07-14T11:12:24.673 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12216 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Jul 14 11:12 1247 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1247_fsm 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1247_vm 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Jul 14 11:12 1249 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1249_fsm 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1249_vm 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Jul 14 11:12 1255 2023-07-14T11:12:24.674 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1255_fsm 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1255_vm 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Jul 14 11:12 1259 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1259_fsm 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1259_vm 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417_vm 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418 2023-07-14T11:12:24.675 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418_vm 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 174 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 175 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2187 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224_vm 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328_vm 2023-07-14T11:12:24.676 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336_vm 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2337 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2600 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2600_fsm 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2600_vm 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2601_fsm 2023-07-14T11:12:24.677 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601_vm 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2602 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2602_fsm 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2602_vm 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2603 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2603_fsm 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2603_vm 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604 2023-07-14T11:12:24.678 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604_vm 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2605 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2605_fsm 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2605_vm 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2606_fsm 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606_vm 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607 2023-07-14T11:12:24.679 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607_fsm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2607_vm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Jul 14 11:12 2608 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2608_fsm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2608_vm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Jul 14 11:12 2609 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2609_fsm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2609_vm 2023-07-14T11:12:24.680 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610_fsm 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2610_vm 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611_vm 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2612_fsm 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612_vm 2023-07-14T11:12:24.681 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613_vm 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2615_fsm 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615_vm 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616_fsm 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2616_vm 2023-07-14T11:12:24.682 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Jul 14 11:12 2617 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2617_fsm 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2617_vm 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Jul 14 11:12 2618 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2618_fsm 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2618_vm 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Jul 14 11:12 2619 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2619_fsm 2023-07-14T11:12:24.683 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2619_vm 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620_vm 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2650 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2651 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2652 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2653 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2654 2023-07-14T11:12:24.684 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2655 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2656 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2657 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Jul 14 11:12 2658 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2659 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2660 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2661 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2662 2023-07-14T11:12:24.685 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2663 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2664 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2665 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2666 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2667 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2668 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2669 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2670 2023-07-14T11:12:24.686 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 335872 Jul 14 11:12 2673 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Jul 14 11:12 2674 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Jul 14 11:12 2675 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2678 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2679 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2680 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2681 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2682 2023-07-14T11:12:24.687 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2683 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2684 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2685 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2686 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2687 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2688 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2689 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2690 2023-07-14T11:12:24.688 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Jul 14 11:12 2691 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2692 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2693 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2696 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2699 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2701 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2702 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2703 2023-07-14T11:12:24.689 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2704 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2753 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2753_fsm 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2753_vm 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2754 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2755 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2756 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2757 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830 2023-07-14T11:12:24.690 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830_vm 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2831 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832_vm 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2833 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834_vm 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2835 2023-07-14T11:12:24.691 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836_vm 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2837 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Jul 14 11:12 2838 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2838_fsm 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2838_vm 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2839 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840 2023-07-14T11:12:24.692 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840_fsm 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2840_vm 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2841 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995_vm 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2996 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079 2023-07-14T11:12:24.693 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3079_fsm 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079_vm 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3080 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3081 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3085 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118_vm 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3119 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3164 2023-07-14T11:12:24.694 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256_vm 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3257 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3258 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350_vm 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3351 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3379 2023-07-14T11:12:24.695 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3380 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381_vm 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3394 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3394_fsm 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3394_vm 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3395 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439 2023-07-14T11:12:24.696 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439_vm 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3440 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 3455 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3456 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3456_fsm 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3456_vm 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466_vm 2023-07-14T11:12:24.697 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3467 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3468 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501_vm 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3502 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3503 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3534 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541 2023-07-14T11:12:24.698 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3541_fsm 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541_vm 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3542 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3574 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3575 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576_vm 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596 2023-07-14T11:12:24.699 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596_vm 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3597 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598_vm 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3599 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3600_fsm 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600_vm 2023-07-14T11:12:24.700 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3601_fsm 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601_vm 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3602_fsm 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602_vm 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3603 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3603_fsm 2023-07-14T11:12:24.701 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3603_vm 2023-07-14T11:12:24.702 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3604 2023-07-14T11:12:24.702 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3605 2023-07-14T11:12:24.702 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3606 2023-07-14T11:12:24.704 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3607 2023-07-14T11:12:24.704 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3608 2023-07-14T11:12:24.704 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 3609 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3712 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3764_fsm 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764_vm 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3766 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3767 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3997 2023-07-14T11:12:24.705 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 5002 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 548 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 549 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102_vm 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104_vm 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106 2023-07-14T11:12:24.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106_vm 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6110 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6111 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6112 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6113 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6117 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826_vm 2023-07-14T11:12:24.707 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 827 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 828 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Jul 14 11:12 pg_filenode.map 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 PG_VERSION 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/base/12346: 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:total 7217 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 . 2023-07-14T11:12:24.708 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 .. 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 112 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 113 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Jul 14 11:12 12182 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12182_fsm 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12182_vm 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12184 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12186 2023-07-14T11:12:24.709 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12187_fsm 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187_vm 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12189 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12191 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12192_fsm 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192_vm 2023-07-14T11:12:24.710 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12194 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12196 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12197_fsm 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197_vm 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12199 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12201 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202 2023-07-14T11:12:24.711 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12202_fsm 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202_vm 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12204 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12206 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12207_fsm 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207_vm 2023-07-14T11:12:24.712 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12209 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12211 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12212 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12214 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12216 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Jul 14 11:12 1247 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1247_fsm 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1247_vm 2023-07-14T11:12:24.713 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Jul 14 11:12 1249 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1249_fsm 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1249_vm 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Jul 14 11:12 1255 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1255_fsm 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1255_vm 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Jul 14 11:12 1259 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1259_fsm 2023-07-14T11:12:24.714 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1259_vm 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417_vm 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418_vm 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 174 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 175 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2187 2023-07-14T11:12:24.715 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224_vm 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328_vm 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336_vm 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2337 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2600 2023-07-14T11:12:24.716 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2600_fsm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2600_vm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2601_fsm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601_vm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2602 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2602_fsm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2602_vm 2023-07-14T11:12:24.717 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2603 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2603_fsm 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2603_vm 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604_vm 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2605 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2605_fsm 2023-07-14T11:12:24.718 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2605_vm 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2606_fsm 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606_vm 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607_fsm 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2607_vm 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Jul 14 11:12 2608 2023-07-14T11:12:24.719 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2608_fsm 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2608_vm 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Jul 14 11:12 2609 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2609_fsm 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2609_vm 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610 2023-07-14T11:12:24.720 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610_fsm 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2610_vm 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611_vm 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2612_fsm 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612_vm 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613 2023-07-14T11:12:24.721 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613_vm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2615_fsm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615_vm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616_fsm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2616_vm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Jul 14 11:12 2617 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2617_fsm 2023-07-14T11:12:24.722 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2617_vm 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Jul 14 11:12 2618 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2618_fsm 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2618_vm 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Jul 14 11:12 2619 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2619_fsm 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2619_vm 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620 2023-07-14T11:12:24.723 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620_vm 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2650 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2651 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2652 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2653 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2654 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2655 2023-07-14T11:12:24.724 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2656 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2657 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Jul 14 11:12 2658 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2659 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2660 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2661 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2662 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2663 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2664 2023-07-14T11:12:24.725 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2665 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2666 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2667 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2668 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2669 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2670 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 335872 Jul 14 11:12 2673 2023-07-14T11:12:24.726 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Jul 14 11:12 2674 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Jul 14 11:12 2675 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2678 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2679 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2680 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2681 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2682 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2683 2023-07-14T11:12:24.727 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2684 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2685 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2686 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2687 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2688 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2689 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2690 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Jul 14 11:12 2691 2023-07-14T11:12:24.728 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2692 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2693 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2696 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2699 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2701 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2702 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2703 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2704 2023-07-14T11:12:24.729 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2753 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2753_fsm 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2753_vm 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2754 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2755 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2756 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2757 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830 2023-07-14T11:12:24.730 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830_vm 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2831 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832_vm 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2833 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834_vm 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2835 2023-07-14T11:12:24.731 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836_vm 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2837 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Jul 14 11:12 2838 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2838_fsm 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2838_vm 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2839 2023-07-14T11:12:24.732 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840_fsm 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2840_vm 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2841 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995_vm 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2996 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079 2023-07-14T11:12:24.733 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3079_fsm 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079_vm 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3080 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3081 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3085 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118_vm 2023-07-14T11:12:24.734 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3119 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3164 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256_vm 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3257 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3258 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350_vm 2023-07-14T11:12:24.735 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3351 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3379 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3380 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381_vm 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3394 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3394_fsm 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3394_vm 2023-07-14T11:12:24.736 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3395 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439_vm 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3440 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 3455 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3456 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3456_fsm 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3456_vm 2023-07-14T11:12:24.737 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466_vm 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3467 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3468 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501_vm 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3502 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3503 2023-07-14T11:12:24.738 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3534 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3541_fsm 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541_vm 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3542 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3574 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3575 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576 2023-07-14T11:12:24.739 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576_vm 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596_vm 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3597 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598_vm 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3599 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600 2023-07-14T11:12:24.740 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3600_fsm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600_vm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3601_fsm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601_vm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3602_fsm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602_vm 2023-07-14T11:12:24.741 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3603 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3603_fsm 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3603_vm 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3604 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3605 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3606 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3607 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3608 2023-07-14T11:12:24.742 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 3609 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3712 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3764_fsm 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764_vm 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3766 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3767 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3997 2023-07-14T11:12:24.743 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 5002 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 548 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 549 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102_vm 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104_vm 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106 2023-07-14T11:12:24.744 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106_vm 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6110 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6111 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6112 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6113 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6117 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826_vm 2023-07-14T11:12:24.745 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 827 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 828 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Jul 14 11:12 pg_filenode.map 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 PG_VERSION 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/base/12347: 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:total 7217 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 293 Jul 14 11:12 . 2023-07-14T11:12:24.746 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 5 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 .. 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 112 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 113 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 57344 Jul 14 11:12 12182 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12182_fsm 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12182_vm 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12184 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12186 2023-07-14T11:12:24.747 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12187_fsm 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12187_vm 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12189 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12191 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12192_fsm 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12192_vm 2023-07-14T11:12:24.748 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12194 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12196 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12197_fsm 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12197_vm 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12199 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12201 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202 2023-07-14T11:12:24.749 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12202_fsm 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12202_vm 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12204 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12206 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 12207_fsm 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12207_vm 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12209 2023-07-14T11:12:24.750 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12211 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12212 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 12214 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 12216 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 73728 Jul 14 11:12 1247 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1247_fsm 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1247_vm 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 393216 Jul 14 11:12 1249 2023-07-14T11:12:24.751 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1249_fsm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1249_vm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 606208 Jul 14 11:12 1255 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1255_fsm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1255_vm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 90112 Jul 14 11:12 1259 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1259_fsm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1259_vm 2023-07-14T11:12:24.752 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1417_vm 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 1418_vm 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 174 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 175 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2187 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224 2023-07-14T11:12:24.753 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2224_vm 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2328_vm 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2336_vm 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2337 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2600 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2600_fsm 2023-07-14T11:12:24.754 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2600_vm 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2601_fsm 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2601_vm 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2602 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2602_fsm 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2602_vm 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2603 2023-07-14T11:12:24.755 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2603_fsm 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2603_vm 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2604_vm 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2605 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2605_fsm 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2605_vm 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606 2023-07-14T11:12:24.756 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2606_fsm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2606_vm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2607_fsm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2607_vm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 442368 Jul 14 11:12 2608 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2608_fsm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2608_vm 2023-07-14T11:12:24.757 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 286720 Jul 14 11:12 2609 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2609_fsm 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2609_vm 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2610_fsm 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2610_vm 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2611_vm 2023-07-14T11:12:24.758 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2612_fsm 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2612_vm 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2613_vm 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2615_fsm 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2615_vm 2023-07-14T11:12:24.759 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2616_fsm 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2616_vm 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 122880 Jul 14 11:12 2617 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2617_fsm 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2617_vm 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 98304 Jul 14 11:12 2618 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2618_fsm 2023-07-14T11:12:24.760 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2618_vm 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 131072 Jul 14 11:12 2619 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2619_fsm 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2619_vm 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2620_vm 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2650 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2651 2023-07-14T11:12:24.761 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2652 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2653 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2654 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2655 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2656 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2657 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 106496 Jul 14 11:12 2658 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2659 2023-07-14T11:12:24.762 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2660 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2661 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2662 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2663 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2664 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2665 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2666 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2667 2023-07-14T11:12:24.763 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2668 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2669 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2670 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 335872 Jul 14 11:12 2673 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 385024 Jul 14 11:12 2674 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 180224 Jul 14 11:12 2675 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2678 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2679 2023-07-14T11:12:24.764 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2680 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2681 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2682 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2683 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2684 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2685 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2686 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2687 2023-07-14T11:12:24.765 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2688 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 49152 Jul 14 11:12 2689 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 81920 Jul 14 11:12 2690 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 270336 Jul 14 11:12 2691 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2692 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2693 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2696 2023-07-14T11:12:24.766 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2699 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2701 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2702 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2703 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 2704 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2753 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2753_fsm 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2753_vm 2023-07-14T11:12:24.767 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2754 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2755 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2756 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 2757 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2830_vm 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2831 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832 2023-07-14T11:12:24.768 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2832_vm 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2833 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2834_vm 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2835 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2836_vm 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2837 2023-07-14T11:12:24.769 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 417792 Jul 14 11:12 2838 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2838_fsm 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2838_vm 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2839 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2840_fsm 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2840_vm 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2841 2023-07-14T11:12:24.770 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2995_vm 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2996 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3079_fsm 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3079_vm 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3080 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3081 2023-07-14T11:12:24.771 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3085 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3118_vm 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3119 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3164 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3256_vm 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3257 2023-07-14T11:12:24.772 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3258 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3350_vm 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3351 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3379 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3380 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3381_vm 2023-07-14T11:12:24.773 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3394 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3394_fsm 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3394_vm 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3395 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3439_vm 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3440 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 40960 Jul 14 11:12 3455 2023-07-14T11:12:24.774 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3456 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3456_fsm 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3456_vm 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3466_vm 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3467 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3468 2023-07-14T11:12:24.775 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3501_vm 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3502 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3503 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3534 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3541_fsm 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3541_vm 2023-07-14T11:12:24.776 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3542 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3574 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3575 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3576_vm 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3596_vm 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3597 2023-07-14T11:12:24.777 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3598_vm 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3599 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3600_fsm 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3600_vm 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601 2023-07-14T11:12:24.778 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3601_fsm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3601_vm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3602_fsm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3602_vm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3603 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3603_fsm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3603_vm 2023-07-14T11:12:24.779 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3604 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3605 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3606 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3607 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3608 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 32768 Jul 14 11:12 3609 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3712 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764 2023-07-14T11:12:24.780 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 3764_fsm 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3764_vm 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3766 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 3767 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3997 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 5002 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 548 2023-07-14T11:12:24.781 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 549 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6102_vm 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6104_vm 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6106_vm 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6110 2023-07-14T11:12:24.782 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6111 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6112 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6113 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6117 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 826_vm 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 827 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 828 2023-07-14T11:12:24.783 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Jul 14 11:12 pg_filenode.map 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 PG_VERSION 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/global: 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:total 553 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 58 Jul 14 11:12 . 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1136 2023-07-14T11:12:24.784 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1136_fsm 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1136_vm 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 1137 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1213 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1213_fsm 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1213_vm 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1214 2023-07-14T11:12:24.785 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1214_fsm 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1214_vm 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 1232 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 1233 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1260 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1260_fsm 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1260_vm 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1261 2023-07-14T11:12:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1261_fsm 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1261_vm 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1262 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 1262_fsm 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 1262_vm 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2396 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 24576 Jul 14 11:12 2396_fsm 2023-07-14T11:12:24.787 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2396_vm 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2397 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2671 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2672 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2676 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2677 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2694 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2695 2023-07-14T11:12:24.788 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2697 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16384 Jul 14 11:12 2698 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2846 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2846_vm 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2847 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2964 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2964_vm 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2965 2023-07-14T11:12:24.789 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2966 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 2966_vm 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 2967 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3592 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 3592_vm 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 3593 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 4060 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 4060_vm 2023-07-14T11:12:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 4061 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6000 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6000_vm 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6001 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6002 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6100 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 6100_vm 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6114 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 6115 2023-07-14T11:12:24.791 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 pg_control 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 512 Jul 14 11:12 pg_filenode.map 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/log: 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_commit_ts: 2023-07-14T11:12:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_dynshmem: 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_logical: 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:total 1 2023-07-14T11:12:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 . 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 mappings 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8 Jul 14 11:12 replorigin_checkpoint 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 snapshots 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_logical/mappings: 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.795 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 .. 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_logical/snapshots: 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 3 Jul 14 11:12 .. 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.796 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_multixact: 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 . 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 members 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 offsets 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_multixact/members: 2023-07-14T11:12:24.797 INFO:teuthology.orchestra.run.smithi040.stdout:total 8 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 .. 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 0000 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_multixact/offsets: 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:total 8 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.798 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 4 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 .. 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 0000 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_notify: 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:total 8 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 0000 2023-07-14T11:12:24.799 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_replslot: 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_serial: 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.800 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_snapshots: 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.801 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_stat: 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_stat_tmp: 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.802 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_subtrans: 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:total 8 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 0000 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.803 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_tblspc: 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_twophase: 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.804 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_wal: 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:total 16384 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 . 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 16777216 Jul 14 11:12 000000010000000000000001 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 archive_status 2023-07-14T11:12:24.805 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_wal/archive_status: 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:total 0 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 0 Jul 14 11:12 . 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 3 postgres postgres system_u:object_r:postgresql_db_t:s0 2 Jul 14 11:12 .. 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/cephfs/postgres/data/pg_xact: 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:total 8 2023-07-14T11:12:24.806 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 2 postgres postgres system_u:object_r:postgresql_db_t:s0 1 Jul 14 11:12 . 2023-07-14T11:12:24.807 INFO:teuthology.orchestra.run.smithi040.stdout:drwx------. 20 postgres postgres system_u:object_r:postgresql_db_t:s0 23 Jul 14 11:12 .. 2023-07-14T11:12:24.807 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-------. 1 postgres postgres system_u:object_r:postgresql_db_t:s0 8192 Jul 14 11:12 0000 2023-07-14T11:12:24.807 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo systemctl start postgresql' 2023-07-14T11:12:25.050 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- pgbench -s 500 -i' 2023-07-14T11:12:25.162 INFO:teuthology.orchestra.run.smithi040.stderr:NOTICE: table "pgbench_history" does not exist, skipping 2023-07-14T11:12:25.169 INFO:teuthology.orchestra.run.smithi040.stderr:NOTICE: table "pgbench_tellers" does not exist, skipping 2023-07-14T11:12:25.173 INFO:teuthology.orchestra.run.smithi040.stderr:NOTICE: table "pgbench_accounts" does not exist, skipping 2023-07-14T11:12:25.179 INFO:teuthology.orchestra.run.smithi040.stderr:NOTICE: table "pgbench_branches" does not exist, skipping 2023-07-14T11:12:25.505 INFO:teuthology.orchestra.run.smithi040.stderr:creating tables... 2023-07-14T11:12:25.613 INFO:teuthology.orchestra.run.smithi040.stderr:100000 of 50000000 tuples (0%) done (elapsed 0.11 s, remaining 52.71 s) 2023-07-14T11:12:25.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:25 smithi130 ceph-mon[129405]: pgmap v11: 129 pgs: 129 active+clean; 29 MiB data, 381 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 2.8 MiB/s wr, 75 op/s 2023-07-14T11:12:25.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:25 smithi040 ceph-mon[97954]: pgmap v11: 129 pgs: 129 active+clean; 29 MiB data, 381 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 2.8 MiB/s wr, 75 op/s 2023-07-14T11:12:25.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:25 smithi045 ceph-mon[128791]: pgmap v11: 129 pgs: 129 active+clean; 29 MiB data, 381 MiB used, 1.0 TiB / 1.0 TiB avail; 180 KiB/s rd, 2.8 MiB/s wr, 75 op/s 2023-07-14T11:12:25.882 INFO:teuthology.orchestra.run.smithi040.stderr:200000 of 50000000 tuples (0%) done (elapsed 0.38 s, remaining 93.45 s) 2023-07-14T11:12:26.206 INFO:teuthology.orchestra.run.smithi040.stderr:300000 of 50000000 tuples (0%) done (elapsed 0.70 s, remaining 115.69 s) 2023-07-14T11:12:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:400000 of 50000000 tuples (0%) done (elapsed 0.82 s, remaining 101.78 s) 2023-07-14T11:12:26.644 INFO:teuthology.orchestra.run.smithi040.stderr:500000 of 50000000 tuples (1%) done (elapsed 1.14 s, remaining 112.58 s) 2023-07-14T11:12:26.802 INFO:teuthology.orchestra.run.smithi040.stderr:600000 of 50000000 tuples (1%) done (elapsed 1.29 s, remaining 106.61 s) 2023-07-14T11:12:27.063 INFO:teuthology.orchestra.run.smithi040.stderr:700000 of 50000000 tuples (1%) done (elapsed 1.56 s, remaining 109.57 s) 2023-07-14T11:12:27.331 INFO:teuthology.orchestra.run.smithi040.stderr:800000 of 50000000 tuples (1%) done (elapsed 1.82 s, remaining 112.14 s) 2023-07-14T11:12:27.512 INFO:teuthology.orchestra.run.smithi040.stderr:900000 of 50000000 tuples (1%) done (elapsed 2.00 s, remaining 109.36 s) 2023-07-14T11:12:27.773 INFO:teuthology.orchestra.run.smithi040.stderr:1000000 of 50000000 tuples (2%) done (elapsed 2.27 s, remaining 111.01 s) 2023-07-14T11:12:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:27 smithi130 ceph-mon[129405]: pgmap v12: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 4.3 MiB/s wr, 180 op/s 2023-07-14T11:12:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:27 smithi040 ceph-mon[97954]: pgmap v12: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 4.3 MiB/s wr, 180 op/s 2023-07-14T11:12:28.052 INFO:teuthology.orchestra.run.smithi040.stderr:1100000 of 50000000 tuples (2%) done (elapsed 2.55 s, remaining 113.15 s) 2023-07-14T11:12:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:27 smithi045 ceph-mon[128791]: pgmap v12: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 4.3 MiB/s wr, 180 op/s 2023-07-14T11:12:28.218 INFO:teuthology.orchestra.run.smithi040.stderr:1200000 of 50000000 tuples (2%) done (elapsed 2.71 s, remaining 110.24 s) 2023-07-14T11:12:28.493 INFO:teuthology.orchestra.run.smithi040.stderr:1300000 of 50000000 tuples (2%) done (elapsed 2.99 s, remaining 111.84 s) 2023-07-14T11:12:28.809 INFO:teuthology.orchestra.run.smithi040.stderr:1400000 of 50000000 tuples (2%) done (elapsed 3.30 s, remaining 114.61 s) 2023-07-14T11:12:28.963 INFO:teuthology.orchestra.run.smithi040.stderr:1500000 of 50000000 tuples (3%) done (elapsed 3.46 s, remaining 111.73 s) 2023-07-14T11:12:29.152 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:12:29.242 INFO:teuthology.orchestra.run.smithi040.stderr:1600000 of 50000000 tuples (3%) done (elapsed 3.73 s, remaining 112.97 s) 2023-07-14T11:12:29.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 -- 172.21.15.40:0/4132113121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c1015b0 msgr2=0x7fdc2c109aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 --2- 172.21.15.40:0/4132113121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c109aa0 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7fdc1c009990 tx=0x7fdc1c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 -- 172.21.15.40:0/4132113121 shutdown_connections 2023-07-14T11:12:29.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 --2- 172.21.15.40:0/4132113121 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c10a070 0x7fdc2c10c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 --2- 172.21.15.40:0/4132113121 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 --2- 172.21.15.40:0/4132113121 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c109aa0 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 -- 172.21.15.40:0/4132113121 >> 172.21.15.40:0/4132113121 conn(0x7fdc2c0fa4b0 msgr2=0x7fdc2c0fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 -- 172.21.15.40:0/4132113121 shutdown_connections 2023-07-14T11:12:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.401+0000 7fdc333ff700 1 -- 172.21.15.40:0/4132113121 wait complete. 2023-07-14T11:12:29.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 Processor -- start 2023-07-14T11:12:29.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 -- start start 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c0fee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c10a070 0x7fdc2c1a3dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc2c0ff660 con 0x7fdc2c1015b0 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc2c1a4310 con 0x7fdc2c100ca0 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.403+0000 7fdc333ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc2c1a4450 con 0x7fdc2c10a070 2023-07-14T11:12:29.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59288/0 (socket says 172.21.15.40:59288) 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 -- 172.21.15.40:0/3490671803 learned_addr learned my addr 172.21.15.40:0/3490671803 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3099a700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c0fee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3199c700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c10a070 0x7fdc2c1a3dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 -- 172.21.15.40:0/3490671803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c10a070 msgr2=0x7fdc2c1a3dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c10a070 0x7fdc2c1a3dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 -- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c1015b0 msgr2=0x7fdc2c0fee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c0fee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 -- 172.21.15.40:0/3490671803 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdc1c0095f0 con 0x7fdc2c100ca0 2023-07-14T11:12:29.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc3119b700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fdc2800e9a0 tx=0x7fdc2800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.404+0000 7fdc227fc700 1 -- 172.21.15.40:0/3490671803 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc28009790 con 0x7fdc2c100ca0 2023-07-14T11:12:29.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.405+0000 7fdc227fc700 1 -- 172.21.15.40:0/3490671803 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdc28010430 con 0x7fdc2c100ca0 2023-07-14T11:12:29.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.405+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdc2c1a46b0 con 0x7fdc2c100ca0 2023-07-14T11:12:29.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.405+0000 7fdc227fc700 1 -- 172.21.15.40:0/3490671803 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc28018680 con 0x7fdc2c100ca0 2023-07-14T11:12:29.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.405+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdc2c1a4b90 con 0x7fdc2c100ca0 2023-07-14T11:12:29.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.408+0000 7fdc227fc700 1 -- 172.21.15.40:0/3490671803 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7fdc280105a0 con 0x7fdc2c100ca0 2023-07-14T11:12:29.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.409+0000 7fdc46ceeb80 1 Processor -- start 2023-07-14T11:12:29.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.409+0000 7fdc227fc700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 0x7fdc180a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.409+0000 7fdc46ceeb80 1 -- start start 2023-07-14T11:12:29.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc227fc700 1 -- 172.21.15.40:0/3490671803 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fdc28014070 con 0x7fdc2c100ca0 2023-07-14T11:12:29.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243ede36a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3199c700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 0x7fdc180a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 0x56243edf0eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ece9cb0 con 0x56243edeead0 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ed3ca40 con 0x56243ede3be0 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ed35ed0 con 0x56243ede32d0 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3099a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3099a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59298/0 (socket says 172.21.15.40:59298) 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3099a700 1 -- 172.21.15.40:0/1127212926 learned_addr learned my addr 172.21.15.40:0/1127212926 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243ede36a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.410+0000 7fdc3199c700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 0x56243edf0eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 -- 172.21.15.40:0/1127212926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 msgr2=0x56243ede36a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243ede36a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 -- 172.21.15.40:0/1127212926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 msgr2=0x56243edf0eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 0x56243edf0eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 -- 172.21.15.40:0/1127212926 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56243eb63de0 con 0x56243ede3be0 2023-07-14T11:12:29.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3199c700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 0x56243edf0eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3199c700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 0x7fdc180a3800 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fdc2c100a00 tx=0x7fdc2400fce0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fdc1c000c00 tx=0x7fdc1c00b610 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2b740e70f2f0af89 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc177fe700 1 -- 172.21.15.40:0/1127212926 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc1c01d070 con 0x56243ede3be0 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc177fe700 1 -- 172.21.15.40:0/1127212926 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdc1c01e040 con 0x56243ede3be0 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.411+0000 7fdc177fe700 1 -- 172.21.15.40:0/1127212926 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc1c004900 con 0x56243ede3be0 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1127212926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 msgr2=0x56243edee590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7fdc1c000c00 tx=0x7fdc1c00b610 comp rx=0 tx=0).stop 2023-07-14T11:12:29.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1127212926 shutdown_connections 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243edeead0 0x56243edf0eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243ede36a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1127212926 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243edee590 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1127212926 >> 172.21.15.40:0/1127212926 conn(0x56243edd9da0 msgr2=0x56243eddf6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1127212926 shutdown_connections 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.412+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1127212926 wait complete. 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.413+0000 7fdc46ceeb80 1 -- start start 2023-07-14T11:12:29.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.413+0000 7fdc46ceeb80 1 Processor -- start 2023-07-14T11:12:29.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 0x56243edeb770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243edebe10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ecd5ac0 con 0x56243ede3be0 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ecd57c0 con 0x56243edeead0 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ebb9920 con 0x56243ede32d0 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc46ceeb80 10 client.0 _ll_register_callbacks cb 0x56243ecd3760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:29.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3099a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 0x56243edeb770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58316/0 (socket says 172.21.15.40:58316) 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3119b700 1 -- 172.21.15.40:0/1190053061 learned_addr learned my addr 172.21.15.40:0/1190053061 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3199c700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243edebe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.414+0000 7fdc3119b700 1 -- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 msgr2=0x56243edebe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243edebe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3119b700 1 -- 172.21.15.40:0/1190053061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 msgr2=0x56243edeb770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 0x56243edeb770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3119b700 1 -- 172.21.15.40:0/1190053061 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdc2c10d170 con 0x56243ede32d0 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 0x56243edeb770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fdc28096a50 tx=0x7fdc2809a050 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d8d36d1e5df79537 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc14ff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc2809a440 con 0x56243ede32d0 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56243ec8f4f0 con 0x56243ede32d0 2023-07-14T11:12:29.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.415+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fdc280a2430 con 0x56243ede32d0 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.416+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdc280994d0 con 0x56243ede32d0 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.416+0000 7fdc46ceeb80 10 client.25663 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.416+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x56243eb4cc60 con 0x56243ede32d0 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.416+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56243ec8f4f0 con 0x56243ede32d0 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fdc280a2010 con 0x56243ede32d0 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 10 client.25663 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:29.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 10 client.25663 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:29.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 20 client.25663 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:29.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243edfe330 0x56243ee00710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 4 client.25663 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:29.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.417+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56243ed2eeb0 con 0x56243edfe330 2023-07-14T11:12:29.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.418+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243edfe330 0x56243ee00710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.418+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243edfe330 0x56243ee00710 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=df2813e803c9301a server_cookie=96c5442306e60150 in_seq=0 out_seq=0 2023-07-14T11:12:29.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.418+0000 7fdc14ff9700 10 client.25663 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:29.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1190053061 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x56243ed2eeb0 con 0x56243edfe330 2023-07-14T11:12:29.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc14ff9700 10 client.25663 handle_command_reply: tid=0 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 client.25663 shutdown 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 msgr2=0x56243edee4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7fdc28096a50 tx=0x7fdc2809a050 comp rx=0 tx=0).stop 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 shutdown_connections 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243edfe330 0x56243ee00710 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243edebe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede32d0 0x56243edee4f0 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1190053061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede3be0 0x56243edeb770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc14ff9700 0 client.25663 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:29.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.419+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 >> 172.21.15.40:0/1190053061 conn(0x56243edd9da0 msgr2=0x56243eddf230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.420+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 shutdown_connections 2023-07-14T11:12:29.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.420+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1190053061 wait complete. 2023-07-14T11:12:29.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.420+0000 7fdc46ceeb80 20 client.25663 trim_cache size 0 max 16384 2023-07-14T11:12:29.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.438+0000 7fdc46ceeb80 1 Processor -- start 2023-07-14T11:12:29.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 -- start start 2023-07-14T11:12:29.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243edf54b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede3be0 0x56243edee090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ede4540 con 0x56243ede32d0 2023-07-14T11:12:29.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243edecd50 con 0x56243edeead0 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243edec180 con 0x56243ede3be0 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc3199c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc3199c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59312/0 (socket says 172.21.15.40:59312) 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.439+0000 7fdc3199c700 1 -- 172.21.15.40:0/64504631 learned_addr learned my addr 172.21.15.40:0/64504631 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243edf54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3099a700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede3be0 0x56243edee090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 -- 172.21.15.40:0/64504631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede3be0 msgr2=0x56243edee090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede3be0 0x56243edee090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 -- 172.21.15.40:0/64504631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 msgr2=0x56243ede8a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 -- 172.21.15.40:0/64504631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56243eb8a250 con 0x56243ede32d0 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3199c700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.440+0000 7fdc3119b700 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243edf54b0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fdc28099ce0 tx=0x7fdc28005b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=106c5b1fb1f3c81a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc17fff700 1 -- 172.21.15.40:0/64504631 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc280ca650 con 0x56243ede32d0 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc17fff700 1 -- 172.21.15.40:0/64504631 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdc28099090 con 0x56243ede32d0 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc17fff700 1 -- 172.21.15.40:0/64504631 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc280994d0 con 0x56243ede32d0 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/64504631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 msgr2=0x56243edf54b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243edf54b0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 crypto rx=0x7fdc28099ce0 tx=0x7fdc28005b40 comp rx=0 tx=0).stop 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/64504631 shutdown_connections 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243edeead0 0x56243ede8a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243edf54b0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/64504631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243ede3be0 0x56243edee090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.441+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/64504631 >> 172.21.15.40:0/64504631 conn(0x56243edd9da0 msgr2=0x56243edfa660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.442+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/64504631 shutdown_connections 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.442+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/64504631 wait complete. 2023-07-14T11:12:29.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.442+0000 7fdc46ceeb80 1 -- start start 2023-07-14T11:12:29.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 Processor -- start 2023-07-14T11:12:29.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243ee46310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 0x56243ee511a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ee516e0 con 0x56243ede32d0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ee51820 con 0x56243ede3be0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56243ee519e0 con 0x56243edeead0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.443+0000 7fdc46ceeb80 10 client.0 _ll_register_callbacks cb 0x7fdc28099bd0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243ee46310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59328/0 (socket says 172.21.15.40:59328) 2023-07-14T11:12:29.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3119b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243ee46310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58866/0 (socket says 172.21.15.40:58866) 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 -- 172.21.15.40:0/1576728478 learned_addr learned my addr 172.21.15.40:0/1576728478 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3199c700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 0x56243ee511a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 -- 172.21.15.40:0/1576728478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 msgr2=0x56243ee511a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 0x56243ee511a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 -- 172.21.15.40:0/1576728478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 msgr2=0x56243ee46310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243ee46310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 -- 172.21.15.40:0/1576728478 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56243eb4cc60 con 0x56243ede3be0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3199c700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 0x56243ee511a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc3099a700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fdc1c00b2e0 tx=0x7fdc1c00bb90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e2d37883c7cf581 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.444+0000 7fdc14ff9700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fdc1c01d040 con 0x56243ede3be0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fdc1c01e040 con 0x56243ede3be0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fdc1c034870 con 0x56243ede3be0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fdc1c00bea0 con 0x56243ede3be0 2023-07-14T11:12:29.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 10 client.25670 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x56243eb622b0 con 0x56243ede3be0 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fdc1c01e040 con 0x56243ede3be0 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fdc1c00fb30 con 0x56243ede3be0 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 10 client.25670 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 10 client.25670 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 20 client.25670 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243ee52d80 0x56243ee55190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 4 client.25670 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.445+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56243ecf1050 con 0x56243ee52d80 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243ee52d80 0x56243ee55190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc3119b700 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243ee52d80 0x56243ee55190 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9906157a58c355 server_cookie=70726f5d10128378 in_seq=0 out_seq=0 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc14ff9700 10 client.25670 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc14ff9700 1 -- 172.21.15.40:0/1576728478 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x56243ecf1050 con 0x56243ee52d80 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc14ff9700 10 client.25670 handle_command_reply: tid=0 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.446+0000 7fdc46ceeb80 1 client.25670 shutdown 2023-07-14T11:12:29.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 msgr2=0x56243ee46850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7fdc1c00b2e0 tx=0x7fdc1c00bb90 comp rx=0 tx=0).stop 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 shutdown_connections 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56243ee52d80 0x56243ee55190 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56243edeead0 0x56243ee511a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56243ede32d0 0x56243ee46310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc14ff9700 0 client.25670 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:29.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 --2- 172.21.15.40:0/1576728478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56243ede3be0 0x56243ee46850 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 >> 172.21.15.40:0/1576728478 conn(0x56243edd9da0 msgr2=0x56243edde8b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 shutdown_connections 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 1 -- 172.21.15.40:0/1576728478 wait complete. 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc46ceeb80 20 client.25670 trim_cache size 0 max 16384 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:12:29.457 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "no active scrubs running", 2023-07-14T11:12:29.458 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": {} 2023-07-14T11:12:29.458 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:12:29.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 msgr2=0x7fdc180a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.447+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 0x7fdc180a3800 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fdc2c100a00 tx=0x7fdc2400fce0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 msgr2=0x7fdc2c0fe8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7fdc2800e9a0 tx=0x7fdc2800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 shutdown_connections 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fdc180a1350 0x7fdc180a3800 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fdc2c10a070 0x7fdc2c1a3dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fdc2c100ca0 0x7fdc2c0fe8f0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 --2- 172.21.15.40:0/3490671803 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdc2c1015b0 0x7fdc2c0fee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 >> 172.21.15.40:0/3490671803 conn(0x7fdc2c0fa4b0 msgr2=0x7fdc2c10c170 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 shutdown_connections 2023-07-14T11:12:29.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.448+0000 7fdc333ff700 1 -- 172.21.15.40:0/3490671803 wait complete. 2023-07-14T11:12:29.469 INFO:tasks.cephfs.filesystem:all active scrubs completed 2023-07-14T11:12:29.470 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:12:29.558 INFO:teuthology.orchestra.run.smithi040.stderr:1700000 of 50000000 tuples (3%) done (elapsed 4.05 s, remaining 115.11 s) 2023-07-14T11:12:29.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.700+0000 7f3c387ff700 1 -- 172.21.15.40:0/545466512 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c300ffad0 msgr2=0x7f3c300fff40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.700+0000 7f3c387ff700 1 --2- 172.21.15.40:0/545466512 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c300ffad0 0x7f3c300fff40 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7f3c20009990 tx=0x7f3c20009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 -- 172.21.15.40:0/545466512 shutdown_connections 2023-07-14T11:12:29.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 --2- 172.21.15.40:0/545466512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c30100480 0x7f3c3010c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 --2- 172.21.15.40:0/545466512 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 0x7f3c30105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 --2- 172.21.15.40:0/545466512 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c300ffad0 0x7f3c300fff40 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 -- 172.21.15.40:0/545466512 >> 172.21.15.40:0/545466512 conn(0x7f3c300fb590 msgr2=0x7f3c300fd9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:29.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 -- 172.21.15.40:0/545466512 shutdown_connections 2023-07-14T11:12:29.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.702+0000 7f3c387ff700 1 -- 172.21.15.40:0/545466512 wait complete. 2023-07-14T11:12:29.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.704+0000 7f3c387ff700 1 Processor -- start 2023-07-14T11:12:29.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.704+0000 7f3c387ff700 1 -- start start 2023-07-14T11:12:29.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 0x7f3c30197e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 0x7f3c3019c780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c3019cdc0 con 0x7f3c30105ae0 2023-07-14T11:12:29.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c3019cf30 con 0x7f3c30100480 2023-07-14T11:12:29.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c387ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c3019d0f0 con 0x7f3c300ffad0 2023-07-14T11:12:29.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c35d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c35d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59346/0 (socket says 172.21.15.40:59346) 2023-07-14T11:12:29.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c35d9a700 1 -- 172.21.15.40:0/75484726 learned_addr learned my addr 172.21.15.40:0/75484726 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:29.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.705+0000 7f3c3659b700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 0x7f3c30197e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c36d9c700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 0x7f3c3019c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 -- 172.21.15.40:0/75484726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 msgr2=0x7f3c30198390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 -- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 msgr2=0x7f3c3019c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 0x7f3c3019c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 -- 172.21.15.40:0/75484726 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c200095f0 con 0x7f3c300ffad0 2023-07-14T11:12:29.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c35d9a700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:29.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.706+0000 7f3c3659b700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 0x7f3c30197e50 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f3c2c00d950 tx=0x7f3c2c00dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.707+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c2c00f890 con 0x7f3c300ffad0 2023-07-14T11:12:29.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.707+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c3019d3d0 con 0x7f3c300ffad0 2023-07-14T11:12:29.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.707+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c301a4c10 con 0x7f3c300ffad0 2023-07-14T11:12:29.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.707+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3c2c010460 con 0x7f3c300ffad0 2023-07-14T11:12:29.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.707+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3c2c00e610 con 0x7f3c300ffad0 2023-07-14T11:12:29.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.709+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c14005320 con 0x7f3c300ffad0 2023-07-14T11:12:29.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.710+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f3c2c00f9f0 con 0x7f3c300ffad0 2023-07-14T11:12:29.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.713+0000 7f3c277fe700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 0x7f3c1c0a3900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:29.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.713+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3c2c0cd160 con 0x7f3c300ffad0 2023-07-14T11:12:29.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.713+0000 7f3c35d9a700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 0x7f3c1c0a3900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:29.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.714+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3c2c096090 con 0x7f3c300ffad0 2023-07-14T11:12:29.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:29.714+0000 7f3c35d9a700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 0x7f3c1c0a3900 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f3c301993f0 tx=0x7f3c2000b540 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:29.739 INFO:teuthology.orchestra.run.smithi040.stderr:1800000 of 50000000 tuples (3%) done (elapsed 4.23 s, remaining 113.31 s) 2023-07-14T11:12:29.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:29 smithi040 ceph-mon[97954]: pgmap v13: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 220 KiB/s rd, 4.2 MiB/s wr, 148 op/s 2023-07-14T11:12:30.007 INFO:teuthology.orchestra.run.smithi040.stderr:1900000 of 50000000 tuples (3%) done (elapsed 4.50 s, remaining 113.91 s) 2023-07-14T11:12:30.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.017+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f3c14005190 con 0x7f3c300ffad0 2023-07-14T11:12:30.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.018+0000 7f3c277fe700 1 -- 172.21.15.40:0/75484726 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7f3c2c020090 con 0x7f3c300ffad0 2023-07-14T11:12:30.021 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:30.021 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:12:30.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.020+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 msgr2=0x7f3c1c0a3900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.020+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 0x7f3c1c0a3900 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f3c301993f0 tx=0x7f3c2000b540 comp rx=0 tx=0).stop 2023-07-14T11:12:30.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.020+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 msgr2=0x7f3c30197e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.020+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 0x7f3c30197e50 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f3c2c00d950 tx=0x7f3c2c00dc60 comp rx=0 tx=0).stop 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 shutdown_connections 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3c1c0a1450 0x7f3c1c0a3900 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3c30105ae0 0x7f3c3019c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3c300ffad0 0x7f3c30197e50 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 --2- 172.21.15.40:0/75484726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3c30100480 0x7f3c30198390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 >> 172.21.15.40:0/75484726 conn(0x7f3c300fb590 msgr2=0x7f3c300fd0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 shutdown_connections 2023-07-14T11:12:30.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.021+0000 7f3c387ff700 1 -- 172.21.15.40:0/75484726 wait complete. 2023-07-14T11:12:30.025 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:12:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:29 smithi130 ceph-mon[129405]: pgmap v13: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 220 KiB/s rd, 4.2 MiB/s wr, 148 op/s 2023-07-14T11:12:30.044 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:4 damage ls 2023-07-14T11:12:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:29 smithi045 ceph-mon[128791]: pgmap v13: 129 pgs: 129 active+clean; 43 MiB data, 416 MiB used, 1.0 TiB / 1.0 TiB avail; 220 KiB/s rd, 4.2 MiB/s wr, 148 op/s 2023-07-14T11:12:30.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 -- 172.21.15.40:0/474643252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80ff1c0 msgr2=0x7f1ee810be90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/474643252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee810be90 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7f1edc009990 tx=0x7f1edc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 -- 172.21.15.40:0/474643252 shutdown_connections 2023-07-14T11:12:30.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/474643252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee810e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/474643252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80fe820 0x7f1ee80febf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/474643252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee810be90 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 -- 172.21.15.40:0/474643252 >> 172.21.15.40:0/474643252 conn(0x7f1ee80fa4b0 msgr2=0x7f1ee80fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 -- 172.21.15.40:0/474643252 shutdown_connections 2023-07-14T11:12:30.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.274+0000 7f1eed5ff700 1 -- 172.21.15.40:0/474643252 wait complete. 2023-07-14T11:12:30.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.276+0000 7f1eed5ff700 1 Processor -- start 2023-07-14T11:12:30.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.276+0000 7f1eed5ff700 1 -- start start 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.276+0000 7f1eed5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 0x7f1ee8197e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1eed5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee8198340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1eed5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee819c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1eed5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee819cde0 con 0x7f1ee810c3d0 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1eed5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee819cf50 con 0x7f1ee80fe820 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1eed5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1ee819d110 con 0x7f1ee80ff1c0 2023-07-14T11:12:30.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee67fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee8198340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 0x7f1ee8197e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee67fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee8198340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58378/0 (socket says 172.21.15.40:58378) 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1667915848 learned_addr learned my addr 172.21.15.40:0/1667915848 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee819c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/1667915848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 msgr2=0x7f1ee8198340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee8198340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 msgr2=0x7f1ee819c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee819c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/1667915848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1edc0095f0 con 0x7f1ee80fe820 2023-07-14T11:12:30.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.277+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:30.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 0x7f1ee8197e00 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f1ed800d950 tx=0x7f1ed800dc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1ecffff700 1 -- 172.21.15.40:0/1667915848 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ed8009930 con 0x7f1ee80fe820 2023-07-14T11:12:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1ecffff700 1 -- 172.21.15.40:0/1667915848 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1ed8010460 con 0x7f1ee80fe820 2023-07-14T11:12:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1ecffff700 1 -- 172.21.15.40:0/1667915848 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ed800b5d0 con 0x7f1ee80fe820 2023-07-14T11:12:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1ee819d3f0 con 0x7f1ee80fe820 2023-07-14T11:12:30.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.278+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ee81a4c30 con 0x7f1ee80fe820 2023-07-14T11:12:30.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.284+0000 7f1ecffff700 1 -- 172.21.15.40:0/1667915848 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f1ed8010840 con 0x7f1ee80fe820 2023-07-14T11:12:30.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.284+0000 7f1f00e85b80 1 Processor -- start 2023-07-14T11:12:30.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 -- start start 2023-07-14T11:12:30.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x5644192886a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x564419293590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419295eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56441918ecb0 con 0x564419288be0 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644191e1a40 con 0x5644192882d0 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644191daed0 con 0x564419293ad0 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.285+0000 7f1ee77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x5644192886a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x5644192886a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59374/0 (socket says 172.21.15.40:59374) 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee77fe700 1 -- 172.21.15.40:0/1135530293 learned_addr learned my addr 172.21.15.40:0/1135530293 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419295eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ecffff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 0x7f1ed00a38c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x564419293590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1135530293 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 msgr2=0x5644192886a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x5644192886a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1135530293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 msgr2=0x564419293590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x564419293590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1135530293 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564419008de0 con 0x564419293ad0 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ecffff700 1 -- 172.21.15.40:0/1667915848 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1ed80ccd40 con 0x7f1ee80fe820 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 0x7f1ed00a38c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.286+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419295eb0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f1edc00b2e0 tx=0x7f1edc005bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4b91926a56022666 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1ecd7fa700 1 -- 172.21.15.40:0/1135530293 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1edc01d070 con 0x564419293ad0 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1ecd7fa700 1 -- 172.21.15.40:0/1135530293 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1edc01e040 con 0x564419293ad0 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1ecd7fa700 1 -- 172.21.15.40:0/1135530293 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1edc004920 con 0x564419293ad0 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1135530293 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 msgr2=0x564419295eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 0x7f1ed00a38c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1ee00075c0 tx=0x7f1ee000d040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.287+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419295eb0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f1edc00b2e0 tx=0x7f1edc005bc0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1135530293 shutdown_connections 2023-07-14T11:12:30.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419295eb0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x5644192886a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1135530293 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x564419293590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1135530293 >> 172.21.15.40:0/1135530293 conn(0x56441927eda0 msgr2=0x5644192846a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1135530293 shutdown_connections 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1135530293 wait complete. 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 -- start start 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.288+0000 7f1f00e85b80 1 Processor -- start 2023-07-14T11:12:30.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x564419290750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419290e40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56441917aac0 con 0x5644192882d0 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56441917a7c0 con 0x564419288be0 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56441905e920 con 0x564419293ad0 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1f00e85b80 10 client.0 _ll_register_callbacks cb 0x564419178760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:58924/0 (socket says 172.21.15.40:58924) 2023-07-14T11:12:30.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/2895736242 learned_addr learned my addr 172.21.15.40:0/2895736242 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x564419290750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 msgr2=0x564419290e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419290e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/2895736242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 msgr2=0x564419290750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.289+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x564419290750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/2895736242 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1ee810f3f0 con 0x5644192882d0 2023-07-14T11:12:30.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f1ed80ca0f0 tx=0x7f1ed8099830 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d781551279ee7c64 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ecaff5700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1ed8084030 con 0x5644192882d0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1ed80a1d80 con 0x5644192882d0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5644191344f0 con 0x5644192882d0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1ed80b2140 con 0x5644192882d0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1f00e85b80 10 client.15687 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x564418ff1c60 con 0x5644192882d0 2023-07-14T11:12:30.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.290+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5644191344f0 con 0x5644192882d0 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1ed80ca310 con 0x5644192882d0 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 10 client.15687 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 10 client.15687 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 20 client.15687 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192a33b0 0x5644192a5790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 4 client.15687 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:30.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5644191d3eb0 con 0x5644192a33b0 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.291+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192a33b0 0x5644192a5790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.292+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192a33b0 0x5644192a5790 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=716f21f22880628d server_cookie=7d2f2ce0d39be4b1 in_seq=0 out_seq=0 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.292+0000 7f1ecaff5700 10 client.15687 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1ecaff5700 1 -- 172.21.15.40:0/2895736242 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5644191d3eb0 con 0x5644192a33b0 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1ecaff5700 10 client.15687 handle_command_reply: tid=0 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 client.15687 shutdown 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 msgr2=0x564419293540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 crypto rx=0x7f1ed80ca0f0 tx=0x7f1ed8099830 comp rx=0 tx=0).stop 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 shutdown_connections 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192a33b0 0x5644192a5790 crc :-1 s=READY pgs=30 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x564419290e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5644192882d0 0x564419293540 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1ecaff5700 0 client.15687 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/2895736242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x564419290750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.293+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 >> 172.21.15.40:0/2895736242 conn(0x56441927eda0 msgr2=0x564419284230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.294+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 shutdown_connections 2023-07-14T11:12:30.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.294+0000 7f1f00e85b80 1 -- 172.21.15.40:0/2895736242 wait complete. 2023-07-14T11:12:30.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.294+0000 7f1f00e85b80 20 client.15687 trim_cache size 0 max 16384 2023-07-14T11:12:30.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 Processor -- start 2023-07-14T11:12:30.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 -- start start 2023-07-14T11:12:30.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x56441928ba90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x56441928bfd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x564419285d20 con 0x564419288be0 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x564419289540 con 0x5644192882d0 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.309+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x564419280310 con 0x564419293ad0 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x56441928ba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59398/0 (socket says 172.21.15.40:59398) 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/1753605440 learned_addr learned my addr 172.21.15.40:0/1753605440 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x56441928bfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1753605440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 msgr2=0x56441928bfd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x56441928bfd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1753605440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 msgr2=0x564419293070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 -- 172.21.15.40:0/1753605440 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56441902f250 con 0x564419288be0 2023-07-14T11:12:30.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:30.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x56441928ba90 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f1edc005910 tx=0x7f1edc01a040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=42d6d94f4353202 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ecdffb700 1 -- 172.21.15.40:0/1753605440 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1edc01d040 con 0x564419288be0 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.310+0000 7f1ecdffb700 1 -- 172.21.15.40:0/1753605440 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1edc0327f0 con 0x564419288be0 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1ecdffb700 1 -- 172.21.15.40:0/1753605440 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1edc00b390 con 0x564419288be0 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1753605440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 msgr2=0x56441928ba90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x56441928ba90 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 crypto rx=0x7f1edc005910 tx=0x7f1edc01a040 comp rx=0 tx=0).stop 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1753605440 shutdown_connections 2023-07-14T11:12:30.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564419293ad0 0x56441928bfd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5644192882d0 0x564419293070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/1753605440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419288be0 0x56441928ba90 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1753605440 >> 172.21.15.40:0/1753605440 conn(0x56441927eda0 msgr2=0x564419282e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1753605440 shutdown_connections 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.311+0000 7f1f00e85b80 1 -- 172.21.15.40:0/1753605440 wait complete. 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 -- start start 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 Processor -- start 2023-07-14T11:12:30.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 0x56441929a1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x5644192984f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 0x564419298a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56441929aad0 con 0x564419293ad0 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x564419298f70 con 0x564419288be0 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.312+0000 7f1f00e85b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5644192990e0 con 0x5644192882d0 2023-07-14T11:12:30.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1f00e85b80 10 client.0 _ll_register_callbacks cb 0x7f1ed8099540 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 0x56441929a1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x5644192984f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 0x56441929a1c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58428/0 (socket says 172.21.15.40:58428) 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee6ffd700 1 -- 172.21.15.40:0/4210208431 learned_addr learned my addr 172.21.15.40:0/4210208431 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 0x564419298a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 -- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 msgr2=0x56441929a1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 0x56441929a1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 -- 172.21.15.40:0/4210208431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 msgr2=0x564419298a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 0x564419298a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 -- 172.21.15.40:0/4210208431 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x564418ff1c60 con 0x564419288be0 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee77fe700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 0x564419298a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ee67fc700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x5644192984f0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x5644192893d0 tx=0x7f1edc009540 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1f3a1508bf8e1149 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.313+0000 7f1ecaff5700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1edc01d070 con 0x564419288be0 2023-07-14T11:12:30.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1edc01e040 con 0x564419288be0 2023-07-14T11:12:30.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1edc032490 con 0x564419288be0 2023-07-14T11:12:30.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1edc032960 con 0x564419288be0 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 10 client.25694 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x564419008de0 con 0x564419288be0 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1edc01e040 con 0x564419288be0 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1edc03a030 con 0x564419288be0 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 10 client.25694 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 10 client.25694 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:12:30.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 20 client.25694 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.314+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192f8030 0x5644192fa410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1f00e85b80 4 client.25694 mds_command: new command op to 24511 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x564419196050 con 0x5644192f8030 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192f8030 0x5644192fa410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1ee6ffd700 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192f8030 0x5644192fa410 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=617b07a8a27d1c71 server_cookie=545d6afdb0c2448f in_seq=0 out_seq=0 2023-07-14T11:12:30.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1ecaff5700 10 client.25694 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:30.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1ecaff5700 1 -- 172.21.15.40:0/4210208431 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x564419196050 con 0x5644192f8030 2023-07-14T11:12:30.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1ecaff5700 10 client.25694 handle_command_reply: tid=0 2023-07-14T11:12:30.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.315+0000 7f1f00e85b80 1 client.25694 shutdown 2023-07-14T11:12:30.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 msgr2=0x5644192984f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x5644192984f0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x5644192893d0 tx=0x7f1edc009540 comp rx=0 tx=0).stop 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 shutdown_connections 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x5644192f8030 0x5644192fa410 crc :-1 s=READY pgs=31 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564419293ad0 0x564419298a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5644192882d0 0x56441929a1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 --2- 172.21.15.40:0/4210208431 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564419288be0 0x5644192984f0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1ecaff5700 0 client.25694 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:30.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 >> 172.21.15.40:0/4210208431 conn(0x56441927eda0 msgr2=0x56441928ba90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 shutdown_connections 2023-07-14T11:12:30.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 1 -- 172.21.15.40:0/4210208431 wait complete. 2023-07-14T11:12:30.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1f00e85b80 20 client.25694 trim_cache size 0 max 16384 2023-07-14T11:12:30.328 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 msgr2=0x7f1ed00a38c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 0x7f1ed00a38c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f1ee00075c0 tx=0x7f1ee000d040 comp rx=0 tx=0).stop 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 msgr2=0x7f1ee8197e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.316+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 0x7f1ee8197e00 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7f1ed800d950 tx=0x7f1ed800dc60 comp rx=0 tx=0).stop 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 shutdown_connections 2023-07-14T11:12:30.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1ed00a1410 0x7f1ed00a38c0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1ee810c3d0 0x7f1ee819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1ee80fe820 0x7f1ee8197e00 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 --2- 172.21.15.40:0/1667915848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1ee80ff1c0 0x7f1ee8198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 >> 172.21.15.40:0/1667915848 conn(0x7f1ee80fa4b0 msgr2=0x7f1ee810cdd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 shutdown_connections 2023-07-14T11:12:30.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.317+0000 7f1eed5ff700 1 -- 172.21.15.40:0/1667915848 wait complete. 2023-07-14T11:12:30.337 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:3 damage ls 2023-07-14T11:12:30.355 INFO:teuthology.orchestra.run.smithi040.stderr:2000000 of 50000000 tuples (4%) done (elapsed 4.85 s, remaining 116.34 s) 2023-07-14T11:12:30.462 INFO:teuthology.orchestra.run.smithi040.stderr:2100000 of 50000000 tuples (4%) done (elapsed 4.95 s, remaining 113.01 s) 2023-07-14T11:12:30.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2343033512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28101630 msgr2=0x7f0a28109b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2343033512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28101630 0x7f0a28109b20 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f0a1c009990 tx=0x7f0a1c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2343033512 shutdown_connections 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2343033512 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 0x7f0a2810c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2343033512 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28100d20 0x7f0a281010f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2343033512 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28101630 0x7f0a28109b20 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2343033512 >> 172.21.15.40:0/2343033512 conn(0x7f0a280fa4b0 msgr2=0x7f0a280fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2343033512 shutdown_connections 2023-07-14T11:12:30.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.563+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2343033512 wait complete. 2023-07-14T11:12:30.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.564+0000 7f0a2e1ff700 1 Processor -- start 2023-07-14T11:12:30.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 -- start start 2023-07-14T11:12:30.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 0x7f0a280feeb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 0x7f0a281a3fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a280ff6e0 con 0x7f0a2810a060 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a281a4500 con 0x7f0a28101630 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a2e1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a281a4640 con 0x7f0a28100d20 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a27fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 0x7f0a281a3fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 0x7f0a280feeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58442/0 (socket says 172.21.15.40:58442) 2023-07-14T11:12:30.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.565+0000 7f0a277fe700 1 -- 172.21.15.40:0/2512654096 learned_addr learned my addr 172.21.15.40:0/2512654096 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 -- 172.21.15.40:0/2512654096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 msgr2=0x7f0a280fe970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 -- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 msgr2=0x7f0a281a3fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 0x7f0a281a3fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 -- 172.21.15.40:0/2512654096 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a1c0095f0 con 0x7f0a28101630 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 0x7f0a280feeb0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0a1c0098e0 tx=0x7f0a1c005720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a24ff9700 1 -- 172.21.15.40:0/2512654096 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1c01d070 con 0x7f0a28101630 2023-07-14T11:12:30.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a24ff9700 1 -- 172.21.15.40:0/2512654096 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0a1c022470 con 0x7f0a28101630 2023-07-14T11:12:30.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a281a4840 con 0x7f0a28101630 2023-07-14T11:12:30.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.566+0000 7f0a24ff9700 1 -- 172.21.15.40:0/2512654096 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1c017650 con 0x7f0a28101630 2023-07-14T11:12:30.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.567+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a281a4cc0 con 0x7f0a28101630 2023-07-14T11:12:30.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.571+0000 7f0a24ff9700 1 -- 172.21.15.40:0/2512654096 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f0a1c0225e0 con 0x7f0a28101630 2023-07-14T11:12:30.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.572+0000 7f0a41bf3b80 1 Processor -- start 2023-07-14T11:12:30.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 -- start start 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78d76a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a24ff9700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 0x7f0a100a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78e2ad0 0x55fad78e4eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad77ddcb0 con 0x55fad78d72d0 2023-07-14T11:12:30.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad7830a40 con 0x55fad78d7be0 2023-07-14T11:12:30.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad7829ed0 con 0x55fad78e2ad0 2023-07-14T11:12:30.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a24ff9700 1 -- 172.21.15.40:0/2512654096 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0a1c0ce580 con 0x7f0a28101630 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.573+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59444/0 (socket says 172.21.15.40:59444) 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 -- 172.21.15.40:0/1301875642 learned_addr learned my addr 172.21.15.40:0/1301875642 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a27fff700 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78d76a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 0x7f0a100a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 -- 172.21.15.40:0/1301875642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78e2ad0 msgr2=0x55fad78e4eb0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78e2ad0 0x55fad78e4eb0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 -- 172.21.15.40:0/1301875642 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 msgr2=0x55fad78d76a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78d76a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 -- 172.21.15.40:0/1301875642 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fad7657de0 con 0x55fad78d7be0 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f0a1c09af00 tx=0x7f0a1c09afd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=cbba8234d48e23c9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.574+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 0x7f0a100a3800 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f0a18011550 tx=0x7f0a18009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.575+0000 7f0a0dffb700 1 -- 172.21.15.40:0/1301875642 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1c0bc0f0 con 0x55fad78d7be0 2023-07-14T11:12:30.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.575+0000 7f0a0dffb700 1 -- 172.21.15.40:0/1301875642 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0a1c09ab60 con 0x55fad78d7be0 2023-07-14T11:12:30.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.575+0000 7f0a0dffb700 1 -- 172.21.15.40:0/1301875642 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1c0a3650 con 0x55fad78d7be0 2023-07-14T11:12:30.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.575+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/1301875642 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 msgr2=0x55fad78e2590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.575+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f0a1c09af00 tx=0x7f0a1c09afd0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/1301875642 shutdown_connections 2023-07-14T11:12:30.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78e2ad0 0x55fad78e4eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78d76a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/1301875642 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e2590 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/1301875642 >> 172.21.15.40:0/1301875642 conn(0x55fad78cdda0 msgr2=0x55fad78d36a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/1301875642 shutdown_connections 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/1301875642 wait complete. 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 -- start start 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.576+0000 7f0a41bf3b80 1 Processor -- start 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad78e80b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78e2ad0 0x55fad78e1aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad77c9ac0 con 0x55fad78d7be0 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad77c97c0 con 0x55fad78e2ad0 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad76ad920 con 0x55fad78d72d0 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a41bf3b80 10 client.0 _ll_register_callbacks cb 0x55fad77c7760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58460/0 (socket says 172.21.15.40:58460) 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad78e80b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a277fe700 1 -- 172.21.15.40:0/280950636 learned_addr learned my addr 172.21.15.40:0/280950636 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.577+0000 7f0a27fff700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78e2ad0 0x55fad78e1aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 -- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78e2ad0 msgr2=0x55fad78e1aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78e2ad0 0x55fad78e1aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 -- 172.21.15.40:0/280950636 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 msgr2=0x55fad78e80b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad78e80b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 -- 172.21.15.40:0/280950636 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a281a4cc0 con 0x55fad78d72d0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a277fe700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0a1800ed80 tx=0x7f0a180047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9e39620a99c46606 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a06ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1801cc90 con 0x55fad78d72d0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55fad77834f0 con 0x55fad78d72d0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.578+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0a18023070 con 0x55fad78d72d0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.579+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0a1801cdf0 con 0x55fad78d72d0 2023-07-14T11:12:30.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.579+0000 7f0a41bf3b80 10 client.25696 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.579+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55fad7640c60 con 0x55fad78d72d0 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.579+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55fad77834f0 con 0x55fad78d72d0 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.579+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f0a18027430 con 0x55fad78d72d0 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 10 client.25696 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 10 client.25696 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 20 client.25696 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78f2060 0x55fad78f4470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 4 client.25696 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:30.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55fad7822eb0 con 0x55fad78f2060 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78f2060 0x55fad78f4470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a06ffd700 10 client.25696 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.580+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78f2060 0x55fad78f4470 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=e92d6c25a57ec53 server_cookie=a31aadd9ccaf41dd in_seq=0 out_seq=0 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.581+0000 7f0a06ffd700 1 -- 172.21.15.40:0/280950636 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55fad7822eb0 con 0x55fad78f2060 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.581+0000 7f0a06ffd700 10 client.25696 handle_command_reply: tid=0 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.581+0000 7f0a41bf3b80 1 client.25696 shutdown 2023-07-14T11:12:30.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 msgr2=0x55fad78e7b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f0a1800ed80 tx=0x7f0a180047c0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 shutdown_connections 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78f2060 0x55fad78f4470 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78e2ad0 0x55fad78e1aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad78e7b70 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a06ffd700 0 client.25696 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/280950636 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad78e80b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 >> 172.21.15.40:0/280950636 conn(0x55fad78cdda0 msgr2=0x55fad78d3230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 shutdown_connections 2023-07-14T11:12:30.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/280950636 wait complete. 2023-07-14T11:12:30.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.582+0000 7f0a41bf3b80 20 client.25696 trim_cache size 0 max 16384 2023-07-14T11:12:30.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.597+0000 7f0a41bf3b80 1 Processor -- start 2023-07-14T11:12:30.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.597+0000 7f0a41bf3b80 1 -- start start 2023-07-14T11:12:30.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78dc1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e7740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad78d8330 con 0x55fad78d72d0 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad78d8540 con 0x55fad78d7be0 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad78e0030 con 0x55fad78cdda0 2023-07-14T11:12:30.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58466/0 (socket says 172.21.15.40:58466) 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 -- 172.21.15.40:0/2796125871 learned_addr learned my addr 172.21.15.40:0/2796125871 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a27fff700 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e7740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78dc1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 -- 172.21.15.40:0/2796125871 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 msgr2=0x55fad78e7740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e7740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 -- 172.21.15.40:0/2796125871 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 msgr2=0x55fad78dc1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78dc1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.598+0000 7f0a277fe700 1 -- 172.21.15.40:0/2796125871 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55fad767e250 con 0x55fad78cdda0 2023-07-14T11:12:30.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.599+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f0a18013010 tx=0x7f0a18026390 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7f50fae53dcdc1ae server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.599+0000 7f0a0e7fc700 1 -- 172.21.15.40:0/2796125871 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a18027430 con 0x55fad78cdda0 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.599+0000 7f0a0e7fc700 1 -- 172.21.15.40:0/2796125871 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0a18021070 con 0x55fad78cdda0 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.599+0000 7f0a0e7fc700 1 -- 172.21.15.40:0/2796125871 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a18004830 con 0x55fad78cdda0 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2796125871 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 msgr2=0x55fad78e1a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 crypto rx=0x7f0a18013010 tx=0x7f0a18026390 comp rx=0 tx=0).stop 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2796125871 shutdown_connections 2023-07-14T11:12:30.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78d7be0 0x55fad78e7740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78cdda0 0x55fad78e1a50 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2796125871 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d72d0 0x55fad78dc1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2796125871 >> 172.21.15.40:0/2796125871 conn(0x55fad78cf600 msgr2=0x55fad78e1340 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2796125871 shutdown_connections 2023-07-14T11:12:30.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.600+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2796125871 wait complete. 2023-07-14T11:12:30.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.601+0000 7f0a41bf3b80 1 -- start start 2023-07-14T11:12:30.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.601+0000 7f0a41bf3b80 1 Processor -- start 2023-07-14T11:12:30.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad793a5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad793df60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad793e510 con 0x55fad78d7be0 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad793e680 con 0x55fad78cdda0 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55fad793e840 con 0x55fad78d72d0 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.602+0000 7f0a41bf3b80 10 client.0 _ll_register_callbacks cb 0x7f0a1800e960 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad793a5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59478/0 (socket says 172.21.15.40:59478) 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a26ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad793a5e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58482/0 (socket says 172.21.15.40:58482) 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 -- 172.21.15.40:0/2884423530 learned_addr learned my addr 172.21.15.40:0/2884423530 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a27fff700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad793df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 -- 172.21.15.40:0/2884423530 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 msgr2=0x55fad793a5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad793a5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 -- 172.21.15.40:0/2884423530 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 msgr2=0x55fad793df60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad793df60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 -- 172.21.15.40:0/2884423530 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55fad7640c60 con 0x55fad78cdda0 2023-07-14T11:12:30.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a277fe700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f0a18009cf0 tx=0x7f0a18014000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9ae78510400652b5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.603+0000 7f0a06ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:30.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0a1801cb70 con 0x55fad78cdda0 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0a18023030 con 0x55fad78cdda0 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0a1c09ab60 con 0x55fad78cdda0 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0a1801ccd0 con 0x55fad78cdda0 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a41bf3b80 10 client.25712 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.604+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55fad76562b0 con 0x55fad78cdda0 2023-07-14T11:12:30.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0a1c09ab60 con 0x55fad78cdda0 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f0a18004a00 con 0x55fad78cdda0 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 10 client.25712 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 10 client.25712 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 20 client.25712 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78e2ad0 0x55fad7948eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 4 client.25712 mds_command: new command op to 24521 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:30.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55fad77e5050 con 0x55fad78e2ad0 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.605+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78e2ad0 0x55fad7948eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a06ffd700 10 client.25712 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a26ffd700 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78e2ad0 0x55fad7948eb0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=791ac9c3b700282 server_cookie=8302c18947d75e87 in_seq=0 out_seq=0 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a06ffd700 1 -- 172.21.15.40:0/2884423530 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55fad77e5050 con 0x55fad78e2ad0 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a06ffd700 10 client.25712 handle_command_reply: tid=0 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 client.25712 shutdown 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 msgr2=0x55fad793a0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f0a18009cf0 tx=0x7f0a18014000 comp rx=0 tx=0).stop 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 shutdown_connections 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55fad78e2ad0 0x55fad7948eb0 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55fad78d7be0 0x55fad793df60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55fad78cdda0 0x55fad793a0a0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 --2- 172.21.15.40:0/2884423530 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55fad78d72d0 0x55fad793a5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 >> 172.21.15.40:0/2884423530 conn(0x55fad78cf600 msgr2=0x55fad78e0250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.606+0000 7f0a06ffd700 0 client.25712 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:30.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 shutdown_connections 2023-07-14T11:12:30.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a41bf3b80 1 -- 172.21.15.40:0/2884423530 wait complete. 2023-07-14T11:12:30.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a41bf3b80 20 client.25712 trim_cache size 0 max 16384 2023-07-14T11:12:30.615 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 msgr2=0x7f0a100a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 0x7f0a100a3800 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f0a18011550 tx=0x7f0a18009260 comp rx=0 tx=0).stop 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 msgr2=0x7f0a280feeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.607+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 0x7f0a280feeb0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f0a1c0098e0 tx=0x7f0a1c005720 comp rx=0 tx=0).stop 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 shutdown_connections 2023-07-14T11:12:30.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0a100a1350 0x7f0a100a3800 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a2810a060 0x7f0a281a3fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0a28100d20 0x7f0a280fe970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 --2- 172.21.15.40:0/2512654096 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0a28101630 0x7f0a280feeb0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 >> 172.21.15.40:0/2512654096 conn(0x7f0a280fa4b0 msgr2=0x7f0a2810c250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 shutdown_connections 2023-07-14T11:12:30.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.608+0000 7f0a2e1ff700 1 -- 172.21.15.40:0/2512654096 wait complete. 2023-07-14T11:12:30.630 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2023-07-14T11:12:30.834 INFO:teuthology.orchestra.run.smithi040.stderr:2200000 of 50000000 tuples (4%) done (elapsed 5.33 s, remaining 115.74 s) 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1019739707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e740728b0 msgr2=0x7f3e74072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1019739707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e740728b0 0x7f3e74072c80 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 crypto rx=0x7f3e640098f0 tx=0x7f3e64009c00 comp rx=0 tx=0).stop 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1019739707 shutdown_connections 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1019739707 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e7410c320 0x7f3e7410e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1019739707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e740728b0 0x7f3e74072c80 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1019739707 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740731c0 0x7f3e7410bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1019739707 >> 172.21.15.40:0/1019739707 conn(0x7f3e740fb5e0 msgr2=0x7f3e740fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1019739707 shutdown_connections 2023-07-14T11:12:30.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.906+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1019739707 wait complete. 2023-07-14T11:12:30.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.907+0000 7f3e7a5ff700 1 Processor -- start 2023-07-14T11:12:30.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.907+0000 7f3e7a5ff700 1 -- start start 2023-07-14T11:12:30.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 0x7f3e74198450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e7410c320 0x7f3e7419c8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e7419cef0 con 0x7f3e7410c320 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e7419d060 con 0x7f3e740728b0 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e7a5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e7419d220 con 0x7f3e740731c0 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e78b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e7410c320 0x7f3e7419c8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59500/0 (socket says 172.21.15.40:59500) 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 -- 172.21.15.40:0/1231724737 learned_addr learned my addr 172.21.15.40:0/1231724737 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 0x7f3e74198450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 -- 172.21.15.40:0/1231724737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 msgr2=0x7f3e74198450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 0x7f3e74198450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 -- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e7410c320 msgr2=0x7f3e7419c8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e7410c320 0x7f3e7419c8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e73fff700 1 -- 172.21.15.40:0/1231724737 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e64009520 con 0x7f3e740728b0 2023-07-14T11:12:30.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.908+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 0x7f3e74198450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:30.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f3e64009810 tx=0x7f3e6400fa00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e717fa700 1 -- 172.21.15.40:0/1231724737 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e6401c070 con 0x7f3e740728b0 2023-07-14T11:12:30.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e717fa700 1 -- 172.21.15.40:0/1231724737 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e6400bbd0 con 0x7f3e740728b0 2023-07-14T11:12:30.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e717fa700 1 -- 172.21.15.40:0/1231724737 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e640049c0 con 0x7f3e740728b0 2023-07-14T11:12:30.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e7419d4a0 con 0x7f3e740728b0 2023-07-14T11:12:30.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.909+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e7419d720 con 0x7f3e740728b0 2023-07-14T11:12:30.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.913+0000 7f3e717fa700 1 -- 172.21.15.40:0/1231724737 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f3e64020070 con 0x7f3e740728b0 2023-07-14T11:12:30.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.914+0000 7f3e717fa700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 0x7f3e5c0a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.914+0000 7f3e8df99b80 1 Processor -- start 2023-07-14T11:12:30.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.914+0000 7f3e717fa700 1 -- 172.21.15.40:0/1231724737 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3e640cdd20 con 0x7f3e740728b0 2023-07-14T11:12:30.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 -- start start 2023-07-14T11:12:30.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 0x7f3e5c0a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309c2d0 0x55b73309c6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 0x55b7330a7590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732fa2cb0 con 0x55b73309cbe0 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732ff5a40 con 0x55b73309c2d0 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732feeed0 con 0x55b7330a7ad0 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e78b9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e78b9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58492/0 (socket says 172.21.15.40:58492) 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e78b9c700 1 -- 172.21.15.40:0/3783378384 learned_addr learned my addr 172.21.15.40:0/3783378384 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.915+0000 7f3e73fff700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309c2d0 0x55b73309c6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e737fe700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 0x55b7330a7590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 -- 172.21.15.40:0/3783378384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309c2d0 msgr2=0x55b73309c6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309c2d0 0x55b73309c6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 -- 172.21.15.40:0/3783378384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 msgr2=0x55b7330a7590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 0x55b7330a7590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 -- 172.21.15.40:0/3783378384 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b732e1cde0 con 0x55b7330a7ad0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e737fe700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 0x55b7330a7590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 0x7f3e5c0a3880 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f3e74199520 tx=0x7f3e680116c0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f3e6c011680 tx=0x7f3e6c009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1d8243a23073c8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e5a7fc700 1 -- 172.21.15.40:0/3783378384 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e6c00e040 con 0x55b7330a7ad0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e5a7fc700 1 -- 172.21.15.40:0/3783378384 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e6c00f040 con 0x55b7330a7ad0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.916+0000 7f3e5a7fc700 1 -- 172.21.15.40:0/3783378384 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e6c004850 con 0x55b7330a7ad0 2023-07-14T11:12:30.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3783378384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 msgr2=0x55b7330a9eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7f3e6c011680 tx=0x7f3e6c009920 comp rx=0 tx=0).stop 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3783378384 shutdown_connections 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a9eb0 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309c2d0 0x55b73309c6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3783378384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309cbe0 0x55b7330a7590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3783378384 >> 172.21.15.40:0/3783378384 conn(0x55b733092da0 msgr2=0x55b7330986a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3783378384 shutdown_connections 2023-07-14T11:12:30.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.917+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3783378384 wait complete. 2023-07-14T11:12:30.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.918+0000 7f3e8df99b80 1 -- start start 2023-07-14T11:12:30.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.918+0000 7f3e8df99b80 1 Processor -- start 2023-07-14T11:12:30.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a4730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a4e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732f8eac0 con 0x55b73309c2d0 2023-07-14T11:12:30.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732f8e7c0 con 0x55b73309cbe0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b732e72920 con 0x55b7330a7ad0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.919+0000 7f3e8df99b80 10 client.0 _ll_register_callbacks cb 0x55b732f8c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e737fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a4730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59010/0 (socket says 172.21.15.40:59010) 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 -- 172.21.15.40:0/3635917004 learned_addr learned my addr 172.21.15.40:0/3635917004 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a4e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 -- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 msgr2=0x55b7330a4e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a4e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 -- 172.21.15.40:0/3635917004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 msgr2=0x55b7330a4730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a4730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 -- 172.21.15.40:0/3635917004 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e6c011330 con 0x55b73309c2d0 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a4e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e73fff700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f3e6409a930 tx=0x7f3e6409a560 comp rx=0 tx=0).ready entity=mon.0 client_cookie=52e12b96ae65f89d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.920+0000 7f3e537fe700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e640cb500 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3e6409a9a0 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b732f484f0 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3e6409e9d0 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e8df99b80 10 client.15720 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55b732e05c60 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e640c6080 con 0x55b73309c2d0 2023-07-14T11:12:30.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3e640a28d0 con 0x55b73309c2d0 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e8df99b80 10 client.15720 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.921+0000 7f3e8df99b80 10 client.15720 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.922+0000 7f3e8df99b80 20 client.15720 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.922+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b7330b6fa0 0x55b7330b93b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.922+0000 7f3e8df99b80 4 client.15720 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.922+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b732fe7eb0 con 0x55b7330b6fa0 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.922+0000 7f3e737fe700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b7330b6fa0 0x55b7330b93b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.923+0000 7f3e737fe700 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b7330b6fa0 0x55b7330b93b0 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=9cdd7b802e3ae118 server_cookie=dcaaade7a0576fe7 in_seq=0 out_seq=0 2023-07-14T11:12:30.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.923+0000 7f3e537fe700 10 client.15720 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:30.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.923+0000 7f3e537fe700 1 -- 172.21.15.40:0/3635917004 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55b732fe7eb0 con 0x55b7330b6fa0 2023-07-14T11:12:30.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.923+0000 7f3e537fe700 10 client.15720 handle_command_reply: tid=0 2023-07-14T11:12:30.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 client.15720 shutdown 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 msgr2=0x55b7330a7520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 crypto rx=0x7f3e6409a930 tx=0x7f3e6409a560 comp rx=0 tx=0).stop 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 shutdown_connections 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b7330b6fa0 0x55b7330b93b0 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a4e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330a7520 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e537fe700 0 client.15720 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/3635917004 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a4730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 >> 172.21.15.40:0/3635917004 conn(0x55b733092da0 msgr2=0x55b733098230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 shutdown_connections 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 1 -- 172.21.15.40:0/3635917004 wait complete. 2023-07-14T11:12:30.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.924+0000 7f3e8df99b80 20 client.15720 trim_cache size 0 max 16384 2023-07-14T11:12:30.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 Processor -- start 2023-07-14T11:12:30.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 -- start start 2023-07-14T11:12:30.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a7050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a3fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b73308fef0 con 0x55b73309c2d0 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7330a02c0 con 0x55b73309cbe0 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7330b3ca0 con 0x55b7330a7ad0 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.938+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59014/0 (socket says 172.21.15.40:59014) 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 -- 172.21.15.40:0/1510422769 learned_addr learned my addr 172.21.15.40:0/1510422769 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a3fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a7050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 -- 172.21.15.40:0/1510422769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 msgr2=0x55b7330a3fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a3fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 -- 172.21.15.40:0/1510422769 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 msgr2=0x55b7330a7050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a7050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 -- 172.21.15.40:0/1510422769 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b732e43250 con 0x55b73309c2d0 2023-07-14T11:12:30.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a3fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:30.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a7050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:30.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f3e64009fd0 tx=0x7f3e64005880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=44c29de1f70d9675 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e5affd700 1 -- 172.21.15.40:0/1510422769 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e640caf80 con 0x55b73309c2d0 2023-07-14T11:12:30.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e5affd700 1 -- 172.21.15.40:0/1510422769 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b732f484f0 con 0x55b73309c2d0 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.939+0000 7f3e5affd700 1 -- 172.21.15.40:0/1510422769 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e64099e90 con 0x55b73309c2d0 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1510422769 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 msgr2=0x55b7330b3760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 crypto rx=0x7f3e64009fd0 tx=0x7f3e64005880 comp rx=0 tx=0).stop 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1510422769 shutdown_connections 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b7330a3fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330b3760 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1510422769 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330a7050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1510422769 >> 172.21.15.40:0/1510422769 conn(0x55b733092da0 msgr2=0x55b733096e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1510422769 shutdown_connections 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.940+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1510422769 wait complete. 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 -- start start 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 Processor -- start 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330acc00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b733103e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b7331043e0 con 0x55b73309c2d0 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b733104550 con 0x55b73309cbe0 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.941+0000 7f3e8df99b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b733104710 con 0x55b7330a7ad0 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e8df99b80 10 client.0 _ll_register_callbacks cb 0x55b732f8c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59020/0 (socket says 172.21.15.40:59020) 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 -- 172.21.15.40:0/1475062314 learned_addr learned my addr 172.21.15.40:0/1475062314 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:30.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330acc00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b733103e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 -- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 msgr2=0x55b733103e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b733103e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 -- 172.21.15.40:0/1475062314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 msgr2=0x55b7330acc00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330acc00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e73fff700 1 -- 172.21.15.40:0/1475062314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e68029860 con 0x55b73309c2d0 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.942+0000 7f3e78b9c700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b733103e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:30.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e73fff700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f3e64096ea0 tx=0x7f3e640cb680 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d44777d36efd47df server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e537fe700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3e6409a9a0 con 0x55b73309c2d0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3e6c00f040 con 0x55b73309c2d0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3e6409ae10 con 0x55b73309c2d0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3e6409e9d0 con 0x55b73309c2d0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e8df99b80 10 client.15726 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55b732e1cde0 con 0x55b73309c2d0 2023-07-14T11:12:30.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.943+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3e6c00f040 con 0x55b73309c2d0 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3e64096820 con 0x55b73309c2d0 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 10 client.15726 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 10 client.15726 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 20 client.15726 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b73310b9e0 0x55b73310ddc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 4 client.15726 mds_command: new command op to 24523 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:30.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b732faa050 con 0x55b73310b9e0 2023-07-14T11:12:30.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b73310b9e0 0x55b73310ddc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e737fe700 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b73310b9e0 0x55b73310ddc0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=f87f64e60578ad84 server_cookie=4ee1f7346ed36f3 in_seq=0 out_seq=0 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.944+0000 7f3e537fe700 10 client.15726 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e537fe700 1 -- 172.21.15.40:0/1475062314 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b732faa050 con 0x55b73310b9e0 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e537fe700 10 client.15726 handle_command_reply: tid=0 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 client.15726 shutdown 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 msgr2=0x55b7330ac6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 crypto rx=0x7f3e64096ea0 tx=0x7f3e640cb680 comp rx=0 tx=0).stop 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 shutdown_connections 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55b73310b9e0 0x55b73310ddc0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b7330a7ad0 0x55b733103e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b73309c2d0 0x55b7330ac6c0 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 --2- 172.21.15.40:0/1475062314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b73309cbe0 0x55b7330acc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 >> 172.21.15.40:0/1475062314 conn(0x55b733092da0 msgr2=0x55b7330a3fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e537fe700 0 client.15726 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:30.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 shutdown_connections 2023-07-14T11:12:30.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 1 -- 172.21.15.40:0/1475062314 wait complete. 2023-07-14T11:12:30.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.945+0000 7f3e8df99b80 20 client.15726 trim_cache size 0 max 16384 2023-07-14T11:12:30.956 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 msgr2=0x7f3e5c0a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 0x7f3e5c0a3880 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f3e74199520 tx=0x7f3e680116c0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 msgr2=0x7f3e74197f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f3e64009810 tx=0x7f3e6400fa00 comp rx=0 tx=0).stop 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 shutdown_connections 2023-07-14T11:12:30.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3e5c0a13d0 0x7f3e5c0a3880 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3e7410c320 0x7f3e7419c8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3e740728b0 0x7f3e74197f10 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 --2- 172.21.15.40:0/1231724737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3e740731c0 0x7f3e74198450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 >> 172.21.15.40:0/1231724737 conn(0x7f3e740fb5e0 msgr2=0x7f3e7410e460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 shutdown_connections 2023-07-14T11:12:30.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:30.946+0000 7f3e7a5ff700 1 -- 172.21.15.40:0/1231724737 wait complete. 2023-07-14T11:12:30.966 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:30 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/75484726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:30.968 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2023-07-14T11:12:31.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:30 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/75484726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:31.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:30 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/75484726' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:31.142 INFO:teuthology.orchestra.run.smithi040.stderr:2300000 of 50000000 tuples (4%) done (elapsed 5.64 s, remaining 116.87 s) 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 -- 172.21.15.40:0/845124277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 msgr2=0x7f300c0fff40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 --2- 172.21.15.40:0/845124277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c0fff40 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8009990 tx=0x7f2ff8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 -- 172.21.15.40:0/845124277 shutdown_connections 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 --2- 172.21.15.40:0/845124277 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c10c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 --2- 172.21.15.40:0/845124277 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 0x7f300c105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 --2- 172.21.15.40:0/845124277 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c0fff40 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 -- 172.21.15.40:0/845124277 >> 172.21.15.40:0/845124277 conn(0x7f300c0fb5f0 msgr2=0x7f300c0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 -- 172.21.15.40:0/845124277 shutdown_connections 2023-07-14T11:12:31.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.205+0000 7f3012fff700 1 -- 172.21.15.40:0/845124277 wait complete. 2023-07-14T11:12:31.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.206+0000 7f3012fff700 1 Processor -- start 2023-07-14T11:12:31.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.206+0000 7f3012fff700 1 -- start start 2023-07-14T11:12:31.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c197d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 0x7f300c19c710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f300c19cd50 con 0x7f300c105ae0 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f300c19cec0 con 0x7f300c100480 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3012fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f300c19d080 con 0x7f300c0ffad0 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f3010d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c197d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f301159c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 0x7f300c19c710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59546/0 (socket says 172.21.15.40:59546) 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 -- 172.21.15.40:0/228103280 learned_addr learned my addr 172.21.15.40:0/228103280 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 -- 172.21.15.40:0/228103280 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 msgr2=0x7f300c197d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c197d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 -- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 msgr2=0x7f300c19c710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 0x7f300c19c710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.207+0000 7f300bfff700 1 -- 172.21.15.40:0/228103280 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2ff80095f0 con 0x7f300c100480 2023-07-14T11:12:31.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f300bfff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8000c00 tx=0x7f2ff8005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f3009ffb700 1 -- 172.21.15.40:0/228103280 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff801d070 con 0x7f300c100480 2023-07-14T11:12:31.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f3009ffb700 1 -- 172.21.15.40:0/228103280 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2ff8022470 con 0x7f300c100480 2023-07-14T11:12:31.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f3009ffb700 1 -- 172.21.15.40:0/228103280 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff8017650 con 0x7f300c100480 2023-07-14T11:12:31.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f300c19d300 con 0x7f300c100480 2023-07-14T11:12:31.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.208+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f300c19d580 con 0x7f300c100480 2023-07-14T11:12:31.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.211+0000 7f3009ffb700 1 -- 172.21.15.40:0/228103280 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f2ff80225e0 con 0x7f300c100480 2023-07-14T11:12:31.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.213+0000 7f3026a0bb80 1 Processor -- start 2023-07-14T11:12:31.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.213+0000 7f3026a0bb80 1 -- start start 2023-07-14T11:12:31.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c7d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27b83cb0 con 0x556e27c7d2d0 2023-07-14T11:12:31.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27bd6a40 con 0x556e27c88ad0 2023-07-14T11:12:31.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27bcfed0 con 0x556e27c7dbe0 2023-07-14T11:12:31.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3009ffb700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 0x7f2ffc0a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3010d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3010d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58550/0 (socket says 172.21.15.40:58550) 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.214+0000 7f3010d9b700 1 -- 172.21.15.40:0/2618173132 learned_addr learned my addr 172.21.15.40:0/2618173132 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f301159c700 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c7d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3009ffb700 1 -- 172.21.15.40:0/228103280 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2ff80ce3b0 con 0x7f300c100480 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f300bfff700 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f301159c700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 0x7f2ffc0a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 -- 172.21.15.40:0/2618173132 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 msgr2=0x556e27c8aeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8aeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 -- 172.21.15.40:0/2618173132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 msgr2=0x556e27c7d6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c7d6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 -- 172.21.15.40:0/2618173132 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556e279fdde0 con 0x556e27c7dbe0 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f3010d9b700 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f3000009d00 tx=0x7f3000011720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=23a3205fc6b8e1f5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.215+0000 7f2ff6ffd700 1 -- 172.21.15.40:0/2618173132 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3000004230 con 0x556e27c7dbe0 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f301159c700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 0x7f2ffc0a3880 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3004009ec0 tx=0x7f300400b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f2ff6ffd700 1 -- 172.21.15.40:0/2618173132 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3000004390 con 0x556e27c7dbe0 2023-07-14T11:12:31.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f2ff6ffd700 1 -- 172.21.15.40:0/2618173132 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f30000197a0 con 0x556e27c7dbe0 2023-07-14T11:12:31.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2618173132 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 msgr2=0x556e27c88590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f3000009d00 tx=0x7f3000011720 comp rx=0 tx=0).stop 2023-07-14T11:12:31.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2618173132 shutdown_connections 2023-07-14T11:12:31.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8aeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c7d6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2618173132 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7dbe0 0x556e27c88590 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.216+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2618173132 >> 172.21.15.40:0/2618173132 conn(0x556e27c73da0 msgr2=0x556e27c796a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.217+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2618173132 shutdown_connections 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.217+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2618173132 wait complete. 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.217+0000 7f3026a0bb80 1 -- start start 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.217+0000 7f3026a0bb80 1 Processor -- start 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c884d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c85d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27b6fac0 con 0x556e27c7dbe0 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27b6f7c0 con 0x556e27c88ad0 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27a53920 con 0x556e27c7d2d0 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3026a0bb80 10 client.0 _ll_register_callbacks cb 0x556e27b6d760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f300bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f300bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59046/0 (socket says 172.21.15.40:59046) 2023-07-14T11:12:31.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f300bfff700 1 -- 172.21.15.40:0/931164651 learned_addr learned my addr 172.21.15.40:0/931164651 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.218+0000 7f3010d9b700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c884d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f301159c700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c85d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 -- 172.21.15.40:0/931164651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 msgr2=0x556e27c884d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c884d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 -- 172.21.15.40:0/931164651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 msgr2=0x556e27c85d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c85d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 -- 172.21.15.40:0/931164651 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3000009960 con 0x556e27c7dbe0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f300bfff700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8097530 tx=0x7f2ff8022bc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a1a83d9bd22b1475 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f2febfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff809a720 con 0x556e27c7dbe0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2ff809f060 con 0x556e27c7dbe0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x556e27b294f0 con 0x556e27c7dbe0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.219+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2ff80a33f0 con 0x556e27c7dbe0 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 10 client.15738 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:31.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x556e279e6c60 con 0x556e27c7dbe0 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x556e27b294f0 con 0x556e27c7dbe0 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2ff80cb980 con 0x556e27c7dbe0 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 10 client.15738 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 10 client.15738 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 20 client.15738 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27c97e90 0x556e27c9a270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 4 client.15738 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:31.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x556e27bc8eb0 con 0x556e27c97e90 2023-07-14T11:12:31.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.220+0000 7f3010d9b700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27c97e90 0x556e27c9a270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.221+0000 7f3010d9b700 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27c97e90 0x556e27c9a270 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=bef0436723f44377 server_cookie=735d47c794b394bf in_seq=0 out_seq=0 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.221+0000 7f2febfff700 10 client.15738 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f2febfff700 1 -- 172.21.15.40:0/931164651 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x556e27bc8eb0 con 0x556e27c97e90 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f2febfff700 10 client.15738 handle_command_reply: tid=0 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 client.15738 shutdown 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 msgr2=0x556e27c85750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8097530 tx=0x7f2ff8022bc0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 shutdown_connections 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27c97e90 0x556e27c9a270 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c85d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c884d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f2febfff700 0 client.15738 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/931164651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c85750 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 >> 172.21.15.40:0/931164651 conn(0x556e27c73da0 msgr2=0x556e27c79230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 shutdown_connections 2023-07-14T11:12:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 1 -- 172.21.15.40:0/931164651 wait complete. 2023-07-14T11:12:31.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.222+0000 7f3026a0bb80 20 client.15738 trim_cache size 0 max 16384 2023-07-14T11:12:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.236+0000 7f3026a0bb80 1 Processor -- start 2023-07-14T11:12:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.236+0000 7f3026a0bb80 1 -- start start 2023-07-14T11:12:31.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c88000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c88ad0 0x556e27c80f70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c7e540 con 0x556e27c7d2d0 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c7edd0 con 0x556e27c7dbe0 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c94d40 con 0x556e27c88ad0 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f301159c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c88ad0 0x556e27c80f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59574/0 (socket says 172.21.15.40:59574) 2023-07-14T11:12:31.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 -- 172.21.15.40:0/2866961043 learned_addr learned my addr 172.21.15.40:0/2866961043 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3010d9b700 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c88000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 -- 172.21.15.40:0/2866961043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c88ad0 msgr2=0x556e27c80f70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c88ad0 0x556e27c80f70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 -- 172.21.15.40:0/2866961043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 msgr2=0x556e27c88000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c88000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f300bfff700 1 -- 172.21.15.40:0/2866961043 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556e27a24250 con 0x556e27c7dbe0 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.237+0000 7f3010d9b700 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c88000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.238+0000 7f300bfff700 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f2ff809ad30 tx=0x7f2ff8005c00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=970c6d44858409c8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.238+0000 7f2ff77fe700 1 -- 172.21.15.40:0/2866961043 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff80c9070 con 0x556e27c7dbe0 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.238+0000 7f2ff77fe700 1 -- 172.21.15.40:0/2866961043 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2ff80cb610 con 0x556e27c7dbe0 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.238+0000 7f2ff77fe700 1 -- 172.21.15.40:0/2866961043 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ff80a33f0 con 0x556e27c7dbe0 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2866961043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 msgr2=0x556e27c80a30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f2ff809ad30 tx=0x7f2ff8005c00 comp rx=0 tx=0).stop 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2866961043 shutdown_connections 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c88ad0 0x556e27c80f70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7d2d0 0x556e27c88000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/2866961043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c7dbe0 0x556e27c80a30 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2866961043 >> 172.21.15.40:0/2866961043 conn(0x556e27c73da0 msgr2=0x556e27c85710 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2866961043 shutdown_connections 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- 172.21.15.40:0/2866961043 wait complete. 2023-07-14T11:12:31.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 -- start start 2023-07-14T11:12:31.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.239+0000 7f3026a0bb80 1 Processor -- start 2023-07-14T11:12:31.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c8b620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8bb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c8c0a0 con 0x556e27c7dbe0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c8c210 con 0x556e27c88ad0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556e27c8c380 con 0x556e27c7d2d0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3026a0bb80 10 client.0 _ll_register_callbacks cb 0x7f2ff8097370 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f300bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c8b620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3010d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3010d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58560/0 (socket says 172.21.15.40:58560) 2023-07-14T11:12:31.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f3010d9b700 1 -- 172.21.15.40:0/751205986 learned_addr learned my addr 172.21.15.40:0/751205986 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f300bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c8b620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59072/0 (socket says 172.21.15.40:59072) 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.240+0000 7f301159c700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8bb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 -- 172.21.15.40:0/751205986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 msgr2=0x556e27c8bb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8bb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 -- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 msgr2=0x556e27c8b620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c8b620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 -- 172.21.15.40:0/751205986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x556e279e6c60 con 0x556e27c7d2d0 2023-07-14T11:12:31.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f301159c700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3010d9b700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f3000009230 tx=0x7f3000009440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4571ff58689f2544 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f2febfff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3000003a90 con 0x556e27c7d2d0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x556e27b8b050 con 0x556e27c7d2d0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3000003bf0 con 0x556e27c7d2d0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.241+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3000005040 con 0x556e27c7d2d0 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 10 client.25735 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:31.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x556e279fc2b0 con 0x556e27c7d2d0 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x556e27b8b050 con 0x556e27c7d2d0 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3000011440 con 0x556e27c7d2d0 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 10 client.25735 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 10 client.25735 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 20 client.25735 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27ced120 0x556e27cef530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 4 client.25735 mds_command: new command op to 24565 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x556e27a1ef70 con 0x556e27ced120 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.242+0000 7f300bfff700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27ced120 0x556e27cef530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f300bfff700 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27ced120 0x556e27cef530 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=e7877ee3431c963e server_cookie=dedf133093511444 in_seq=0 out_seq=0 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f2febfff700 10 client.25735 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f2febfff700 1 -- 172.21.15.40:0/751205986 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x556e27a1ef70 con 0x556e27ced120 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f2febfff700 10 client.25735 handle_command_reply: tid=0 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 client.25735 shutdown 2023-07-14T11:12:31.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 msgr2=0x556e27c8f310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f3000009230 tx=0x7f3000009440 comp rx=0 tx=0).stop 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 shutdown_connections 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x556e27ced120 0x556e27cef530 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x556e27c88ad0 0x556e27c8bb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556e27c7d2d0 0x556e27c8f310 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 --2- 172.21.15.40:0/751205986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556e27c7dbe0 0x556e27c8b620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f2febfff700 0 client.25735 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:31.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 >> 172.21.15.40:0/751205986 conn(0x556e27c73da0 msgr2=0x556e27c80a30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.243+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 shutdown_connections 2023-07-14T11:12:31.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3026a0bb80 1 -- 172.21.15.40:0/751205986 wait complete. 2023-07-14T11:12:31.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3026a0bb80 20 client.25735 trim_cache size 0 max 16384 2023-07-14T11:12:31.254 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:31.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 msgr2=0x7f2ffc0a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 0x7f2ffc0a3880 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3004009ec0 tx=0x7f300400b040 comp rx=0 tx=0).stop 2023-07-14T11:12:31.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 msgr2=0x7f300c1982b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f2ff8000c00 tx=0x7f2ff8005740 comp rx=0 tx=0).stop 2023-07-14T11:12:31.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 shutdown_connections 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ffc0a13d0 0x7f2ffc0a3880 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f300c105ae0 0x7f300c19c710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f300c0ffad0 0x7f300c197d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 --2- 172.21.15.40:0/228103280 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f300c100480 0x7f300c1982b0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 >> 172.21.15.40:0/228103280 conn(0x7f300c0fb5f0 msgr2=0x7f300c0fbb70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.244+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 shutdown_connections 2023-07-14T11:12:31.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.245+0000 7f3012fff700 1 -- 172.21.15.40:0/228103280 wait complete. 2023-07-14T11:12:31.257 INFO:teuthology.orchestra.run.smithi040.stderr:2400000 of 50000000 tuples (4%) done (elapsed 5.75 s, remaining 113.97 s) 2023-07-14T11:12:31.265 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2023-07-14T11:12:31.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 -- 172.21.15.40:0/1238634393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb41049e0 msgr2=0x7efdb4104db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 --2- 172.21.15.40:0/1238634393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb41049e0 0x7efdb4104db0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7efda400b0a0 tx=0x7efda400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 -- 172.21.15.40:0/1238634393 shutdown_connections 2023-07-14T11:12:31.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 --2- 172.21.15.40:0/1238634393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb40ff380 0x7efdb410b810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 --2- 172.21.15.40:0/1238634393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb41049e0 0x7efdb4104db0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 --2- 172.21.15.40:0/1238634393 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40fe9d0 0x7efdb40fee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 -- 172.21.15.40:0/1238634393 >> 172.21.15.40:0/1238634393 conn(0x7efdb40fa4b0 msgr2=0x7efdb40fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 -- 172.21.15.40:0/1238634393 shutdown_connections 2023-07-14T11:12:31.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.486+0000 7efdb97ff700 1 -- 172.21.15.40:0/1238634393 wait complete. 2023-07-14T11:12:31.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.487+0000 7efdb97ff700 1 Processor -- start 2023-07-14T11:12:31.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.487+0000 7efdb97ff700 1 -- start start 2023-07-14T11:12:31.491 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 0x7efdb4198470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb41049e0 0x7efdb419c8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdb419cf10 con 0x7efdb41049e0 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdb419d080 con 0x7efdb40ff380 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb97ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efdb419d240 con 0x7efdb40fe9d0 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdabfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 0x7efdb4198470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58572/0 (socket says 172.21.15.40:58572) 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2203618657 learned_addr learned my addr 172.21.15.40:0/2203618657 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.488+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb41049e0 0x7efdb419c8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2203618657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 msgr2=0x7efdb4198470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 0x7efdb4198470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb41049e0 msgr2=0x7efdb419c8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb41049e0 0x7efdb419c8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2203618657 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efda4009d00 con 0x7efdb40fe9d0 2023-07-14T11:12:31.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdabfff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 0x7efdb4198470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:31.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7efda400b780 tx=0x7efda400bd60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb0ff9700 1 -- 172.21.15.40:0/2203618657 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efda400e040 con 0x7efdb40fe9d0 2023-07-14T11:12:31.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efdb419d4c0 con 0x7efdb40fe9d0 2023-07-14T11:12:31.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb0ff9700 1 -- 172.21.15.40:0/2203618657 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efda40094a0 con 0x7efdb40fe9d0 2023-07-14T11:12:31.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb0ff9700 1 -- 172.21.15.40:0/2203618657 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efda40052a0 con 0x7efdb40fe9d0 2023-07-14T11:12:31.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.489+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efdb41a4d60 con 0x7efdb40fe9d0 2023-07-14T11:12:31.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.494+0000 7efdb0ff9700 1 -- 172.21.15.40:0/2203618657 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7efda401d470 con 0x7efdb40fe9d0 2023-07-14T11:12:31.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.495+0000 7efdcd1cbb80 1 Processor -- start 2023-07-14T11:12:31.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.495+0000 7efdcd1cbb80 1 -- start start 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c20590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c22eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947b1bcb0 con 0x560947c152d0 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947b6ea40 con 0x560947c20ad0 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947b67ed0 con 0x560947c15be0 2023-07-14T11:12:31.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdb37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdb0ff9700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 0x7efda00a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdb37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59082/0 (socket says 172.21.15.40:59082) 2023-07-14T11:12:31.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdb37fe700 1 -- 172.21.15.40:0/645831969 learned_addr learned my addr 172.21.15.40:0/645831969 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.500 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.496+0000 7efdabfff700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c22eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb0ff9700 1 -- 172.21.15.40:0/2203618657 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7efda40ce650 con 0x7efdb40fe9d0 2023-07-14T11:12:31.501 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 0x7efda00a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c20590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 -- 172.21.15.40:0/645831969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 msgr2=0x560947c20590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c20590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 -- 172.21.15.40:0/645831969 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 msgr2=0x560947c156a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 -- 172.21.15.40:0/645831969 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560947995de0 con 0x560947c20ad0 2023-07-14T11:12:31.502 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c20590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb37fe700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdabfff700 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c22eb0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7efd9c0114c0 tx=0x7efd9c0117d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=901bd56e66459cce server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 0x7efda00a3880 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7efdac00bd60 tx=0x7efdac0047c0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efda97fa700 1 -- 172.21.15.40:0/645831969 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd9c004230 con 0x560947c20ad0 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efda97fa700 1 -- 172.21.15.40:0/645831969 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efd9c004cf0 con 0x560947c20ad0 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.497+0000 7efda97fa700 1 -- 172.21.15.40:0/645831969 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd9c005630 con 0x560947c20ad0 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/645831969 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 msgr2=0x560947c22eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.503 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c22eb0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7efd9c0114c0 tx=0x7efd9c0117d0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/645831969 shutdown_connections 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c22eb0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c156a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/645831969 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c20590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/645831969 >> 172.21.15.40:0/645831969 conn(0x560947c0bda0 msgr2=0x560947c116a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/645831969 shutdown_connections 2023-07-14T11:12:31.504 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.498+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/645831969 wait complete. 2023-07-14T11:12:31.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.499+0000 7efdcd1cbb80 1 -- start start 2023-07-14T11:12:31.505 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.499+0000 7efdcd1cbb80 1 Processor -- start 2023-07-14T11:12:31.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.499+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.499+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c1d7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c20ad0 0x560947c1de00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947b07ac0 con 0x560947c20ad0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947b077c0 con 0x560947c152d0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5609479eb920 con 0x560947c15be0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdcd1cbb80 10 client.0 _ll_register_callbacks cb 0x560947b05760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdabfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c1d7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59632/0 (socket says 172.21.15.40:59632) 2023-07-14T11:12:31.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2667327388 learned_addr learned my addr 172.21.15.40:0/2667327388 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c20ad0 0x560947c1de00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2667327388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 msgr2=0x560947c1d7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c1d7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2667327388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c20ad0 msgr2=0x560947c1de00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c20ad0 0x560947c1de00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 -- 172.21.15.40:0/2667327388 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efdac00b930 con 0x560947c152d0 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7efda40977d0 tx=0x7efda400bf90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=53a69d87f0c77db7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.500+0000 7efd9a7fc700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efda40ce480 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7efda402d2f0 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560947ac14f0 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7efda409f120 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 10 client.25760 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7efd9c009c90 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560947ac14f0 con 0x560947c152d0 2023-07-14T11:12:31.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7efda409a8e0 con 0x560947c152d0 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 10 client.25760 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 10 client.25760 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 20 client.25760 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c30040 0x560947c32420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 4 client.25760 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.501+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560947b60eb0 con 0x560947c30040 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.502+0000 7efdabfff700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c30040 0x560947c32420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.502+0000 7efdabfff700 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c30040 0x560947c32420 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=21bba35ec4d73ab7 server_cookie=f61040e7f1ed4999 in_seq=0 out_seq=0 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.502+0000 7efd9a7fc700 10 client.25760 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efd9a7fc700 1 -- 172.21.15.40:0/2667327388 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x560947b60eb0 con 0x560947c30040 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efd9a7fc700 10 client.25760 handle_command_reply: tid=0 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 client.25760 shutdown 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 msgr2=0x560947c204b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7efda40977d0 tx=0x7efda400bf90 comp rx=0 tx=0).stop 2023-07-14T11:12:31.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 shutdown_connections 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c30040 0x560947c32420 crc :-1 s=READY pgs=50 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c20ad0 0x560947c1de00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c152d0 0x560947c204b0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2667327388 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c15be0 0x560947c1d7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efd9a7fc700 0 client.25760 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 >> 172.21.15.40:0/2667327388 conn(0x560947c0bda0 msgr2=0x560947c11290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 shutdown_connections 2023-07-14T11:12:31.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.503+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2667327388 wait complete. 2023-07-14T11:12:31.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.504+0000 7efdcd1cbb80 20 client.25760 trim_cache size 0 max 16384 2023-07-14T11:12:31.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.531+0000 7efdcd1cbb80 1 Processor -- start 2023-07-14T11:12:31.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.531+0000 7efdcd1cbb80 1 -- start start 2023-07-14T11:12:31.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c274b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 0x560947c1aa80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c1e400 con 0x560947c152d0 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c08fa0 con 0x560947c15be0 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c0b970 con 0x560947c20ad0 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59636/0 (socket says 172.21.15.40:59636) 2023-07-14T11:12:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 -- 172.21.15.40:0/2975060199 learned_addr learned my addr 172.21.15.40:0/2975060199 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c274b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 0x560947c1aa80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 -- 172.21.15.40:0/2975060199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 msgr2=0x560947c1aa80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 0x560947c1aa80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 -- 172.21.15.40:0/2975060199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 msgr2=0x560947c274b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c274b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdabfff700 1 -- 172.21.15.40:0/2975060199 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56094797ec60 con 0x560947c15be0 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.532+0000 7efdb37fe700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 0x560947c1aa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.533+0000 7efdabfff700 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7efd9c0114c0 tx=0x7efd9c009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1334d27994fc6cd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.533+0000 7efda9ffb700 1 -- 172.21.15.40:0/2975060199 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd9c02ad80 con 0x560947c15be0 2023-07-14T11:12:31.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.533+0000 7efda9ffb700 1 -- 172.21.15.40:0/2975060199 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efd9c005240 con 0x560947c15be0 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.533+0000 7efda9ffb700 1 -- 172.21.15.40:0/2975060199 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efd9c0038e0 con 0x560947c15be0 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2975060199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 msgr2=0x560947c20050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7efd9c0114c0 tx=0x7efd9c009260 comp rx=0 tx=0).stop 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2975060199 shutdown_connections 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c20ad0 0x560947c1aa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c152d0 0x560947c274b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/2975060199 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c15be0 0x560947c20050 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2975060199 >> 172.21.15.40:0/2975060199 conn(0x560947c0bda0 msgr2=0x560947c18af0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2975060199 shutdown_connections 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/2975060199 wait complete. 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 -- start start 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.534+0000 7efdcd1cbb80 1 Processor -- start 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c15be0 0x560947c7c830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c77e30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c78540 con 0x560947c15be0 2023-07-14T11:12:31.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c786b0 con 0x560947c20ad0 2023-07-14T11:12:31.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560947c78870 con 0x560947c152d0 2023-07-14T11:12:31.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdcd1cbb80 10 client.0 _ll_register_callbacks cb 0x7efda4009b80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:31.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdb2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58606/0 (socket says 172.21.15.40:58606) 2023-07-14T11:12:31.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdb2ffd700 1 -- 172.21.15.40:0/1939942521 learned_addr learned my addr 172.21.15.40:0/1939942521 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:31.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdabfff700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c15be0 0x560947c7c830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.535+0000 7efdb37fe700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c77e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 -- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 msgr2=0x560947c77e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c77e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 -- 172.21.15.40:0/1939942521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c15be0 msgr2=0x560947c7c830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c15be0 0x560947c7c830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 -- 172.21.15.40:0/1939942521 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560947995de0 con 0x560947c152d0 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdb2ffd700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7efda4097050 tx=0x7efda40099e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9a5870245ef581ea server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efd9a7fc700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:31.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efda409a9c0 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7efd9c004cf0 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7efda402d2f0 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7efda409ac30 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdcd1cbb80 10 client.25750 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.536+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5609479bc250 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efd9c004cf0 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7efda40a3250 con 0x560947c152d0 2023-07-14T11:12:31.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 10 client.25750 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 10 client.25750 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 20 client.25750 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c84d40 0x560947c87120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 4 client.25750 mds_command: new command op to 24569 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x560947b23050 con 0x560947c84d40 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.537+0000 7efdabfff700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c84d40 0x560947c87120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:31.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdabfff700 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c84d40 0x560947c87120 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cefba08f17e8fb23 server_cookie=731a6385468fd367 in_seq=0 out_seq=0 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efd9a7fc700 10 client.25750 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efd9a7fc700 1 -- 172.21.15.40:0/1939942521 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x560947b23050 con 0x560947c84d40 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efd9a7fc700 10 client.25750 handle_command_reply: tid=0 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 client.25750 shutdown 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 msgr2=0x560947c7c2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7efda4097050 tx=0x7efda40099e0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 shutdown_connections 2023-07-14T11:12:31.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560947c84d40 0x560947c87120 crc :-1 s=READY pgs=51 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560947c20ad0 0x560947c77e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560947c152d0 0x560947c7c2f0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efd9a7fc700 0 client.25750 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 --2- 172.21.15.40:0/1939942521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560947c15be0 0x560947c7c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 >> 172.21.15.40:0/1939942521 conn(0x560947c0bda0 msgr2=0x560947c171a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 shutdown_connections 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.538+0000 7efdcd1cbb80 1 -- 172.21.15.40:0/1939942521 wait complete. 2023-07-14T11:12:31.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdcd1cbb80 20 client.25750 trim_cache size 0 max 16384 2023-07-14T11:12:31.549 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:31.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 msgr2=0x7efda00a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 0x7efda00a3880 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7efdac00bd60 tx=0x7efdac0047c0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 msgr2=0x7efdb4197f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7efda400b780 tx=0x7efda400bd60 comp rx=0 tx=0).stop 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 shutdown_connections 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efda00a13d0 0x7efda00a3880 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efdb41049e0 0x7efdb419c8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efdb40fe9d0 0x7efdb4197f30 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 --2- 172.21.15.40:0/2203618657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efdb40ff380 0x7efdb4198470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:31.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 >> 172.21.15.40:0/2203618657 conn(0x7efdb40fa4b0 msgr2=0x7efdb40fbf20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:31.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 shutdown_connections 2023-07-14T11:12:31.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:31.539+0000 7efdb97ff700 1 -- 172.21.15.40:0/2203618657 wait complete. 2023-07-14T11:12:31.654 INFO:teuthology.orchestra.run.smithi040.stderr:2500000 of 50000000 tuples (5%) done (elapsed 6.15 s, remaining 116.78 s) 2023-07-14T11:12:31.766 INFO:teuthology.orchestra.run.smithi040.stderr:2600000 of 50000000 tuples (5%) done (elapsed 6.26 s, remaining 114.11 s) 2023-07-14T11:12:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:31 smithi130 ceph-mon[129405]: pgmap v14: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 365 KiB/s rd, 33 MiB/s wr, 172 op/s 2023-07-14T11:12:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:31 smithi040 ceph-mon[97954]: pgmap v14: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 365 KiB/s rd, 33 MiB/s wr, 172 op/s 2023-07-14T11:12:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:31 smithi045 ceph-mon[128791]: pgmap v14: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 365 KiB/s rd, 33 MiB/s wr, 172 op/s 2023-07-14T11:12:32.072 INFO:teuthology.orchestra.run.smithi040.stderr:2700000 of 50000000 tuples (5%) done (elapsed 6.56 s, remaining 115.01 s) 2023-07-14T11:12:32.355 INFO:teuthology.orchestra.run.smithi040.stderr:2800000 of 50000000 tuples (5%) done (elapsed 6.85 s, remaining 115.44 s) 2023-07-14T11:12:32.487 INFO:teuthology.orchestra.run.smithi040.stderr:2900000 of 50000000 tuples (5%) done (elapsed 6.98 s, remaining 113.37 s) 2023-07-14T11:12:32.568 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2023-07-14T11:12:32.568 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2023-07-14T11:12:32.781 INFO:teuthology.orchestra.run.smithi040.stderr:3000000 of 50000000 tuples (6%) done (elapsed 7.27 s, remaining 113.96 s) 2023-07-14T11:12:32.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3055225033 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 msgr2=0x7fbd78072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3055225033 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78072c80 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 crypto rx=0x7fbd68009990 tx=0x7fbd68009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3055225033 shutdown_connections 2023-07-14T11:12:32.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3055225033 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7810e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3055225033 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78072c80 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3055225033 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 0x7fbd7810bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3055225033 >> 172.21.15.40:0/3055225033 conn(0x7fbd780fb5f0 msgr2=0x7fbd780fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3055225033 shutdown_connections 2023-07-14T11:12:32.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.804+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3055225033 wait complete. 2023-07-14T11:12:32.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.805+0000 7fbd7e7ff700 1 Processor -- start 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.805+0000 7fbd7e7ff700 1 -- start start 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 0x7fbd78198300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7819c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd7819cda0 con 0x7fbd780728b0 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd7819cf10 con 0x7fbd780731c0 2023-07-14T11:12:32.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7e7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbd7819d0d0 con 0x7fbd7810c310 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59138/0 (socket says 172.21.15.40:59138) 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 -- 172.21.15.40:0/3556081626 learned_addr learned my addr 172.21.15.40:0/3556081626 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 0x7fbd78198300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7819c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 -- 172.21.15.40:0/3556081626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 msgr2=0x7fbd7819c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7819c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 -- 172.21.15.40:0/3556081626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 msgr2=0x7fbd78198300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 0x7fbd78198300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd77fff700 1 -- 172.21.15.40:0/3556081626 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd6c009520 con 0x7fbd780728b0 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.806+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7819c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd77fff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fbd68000c00 tx=0x7fbd68005740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd757fa700 1 -- 172.21.15.40:0/3556081626 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6801d070 con 0x7fbd780728b0 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd757fa700 1 -- 172.21.15.40:0/3556081626 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd68022470 con 0x7fbd780728b0 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbd680095f0 con 0x7fbd780728b0 2023-07-14T11:12:32.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd757fa700 1 -- 172.21.15.40:0/3556081626 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd68017650 con 0x7fbd780728b0 2023-07-14T11:12:32.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.807+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd781a4a10 con 0x7fbd780728b0 2023-07-14T11:12:32.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.810+0000 7fbd757fa700 1 -- 172.21.15.40:0/3556081626 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7fbd680225e0 con 0x7fbd780728b0 2023-07-14T11:12:32.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.812+0000 7fbd757fa700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 0x7fbd600a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.812+0000 7fbd9217cb80 1 Processor -- start 2023-07-14T11:12:32.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.812+0000 7fbd757fa700 1 -- 172.21.15.40:0/3556081626 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fbd680ce640 con 0x7fbd780728b0 2023-07-14T11:12:32.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.812+0000 7fbd9217cb80 1 -- start start 2023-07-14T11:12:32.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.812+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 0x7fbd600a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a4610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff77a4b50 0x558ff77a6f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff769fd30 con 0x558ff77a4b50 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff76f2ac0 con 0x558ff7799350 2023-07-14T11:12:32.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff76ebf50 con 0x558ff7799c60 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59656/0 (socket says 172.21.15.40:59656) 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 -- 172.21.15.40:0/4274888404 learned_addr learned my addr 172.21.15.40:0/4274888404 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 0x7fbd600a3880 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fbd781993d0 tx=0x7fbd6c009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd7cd9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff77a4b50 0x558ff77a6f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd777fe700 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a4610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 -- 172.21.15.40:0/4274888404 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 msgr2=0x558ff77a4610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a4610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 -- 172.21.15.40:0/4274888404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff77a4b50 msgr2=0x558ff77a6f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff77a4b50 0x558ff77a6f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.813+0000 7fbd77fff700 1 -- 172.21.15.40:0/4274888404 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558ff7519e60 con 0x558ff7799350 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.814+0000 7fbd77fff700 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fbd68097d30 tx=0x7fbd68005fd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e77e89ba2b11bd5b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.814+0000 7fbd5e7fc700 1 -- 172.21.15.40:0/4274888404 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6809f2f0 con 0x558ff7799350 2023-07-14T11:12:32.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.814+0000 7fbd5e7fc700 1 -- 172.21.15.40:0/4274888404 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd680a3680 con 0x558ff7799350 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.814+0000 7fbd5e7fc700 1 -- 172.21.15.40:0/4274888404 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd680cbe10 con 0x558ff7799350 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- 172.21.15.40:0/4274888404 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 msgr2=0x558ff7799720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7fbd68097d30 tx=0x7fbd68005fd0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- 172.21.15.40:0/4274888404 shutdown_connections 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff77a4b50 0x558ff77a6f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff7799350 0x558ff7799720 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/4274888404 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a4610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- 172.21.15.40:0/4274888404 >> 172.21.15.40:0/4274888404 conn(0x558ff778fe20 msgr2=0x558ff7795720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- 172.21.15.40:0/4274888404 shutdown_connections 2023-07-14T11:12:32.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- 172.21.15.40:0/4274888404 wait complete. 2023-07-14T11:12:32.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.815+0000 7fbd9217cb80 1 -- start start 2023-07-14T11:12:32.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 Processor -- start 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a9c30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a3af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff768bb40 con 0x558ff7799350 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff768b840 con 0x558ff77a4b50 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff756f9a0 con 0x558ff7799c60 2023-07-14T11:12:32.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd9217cb80 10 client.0 _ll_register_callbacks cb 0x558ff76897e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd77fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a9c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58640/0 (socket says 172.21.15.40:58640) 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.816+0000 7fbd777fe700 1 -- 172.21.15.40:0/3785039179 learned_addr learned my addr 172.21.15.40:0/3785039179 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a3af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 -- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 msgr2=0x558ff77a3af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a3af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 -- 172.21.15.40:0/3785039179 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 msgr2=0x558ff77a9c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a9c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 -- 172.21.15.40:0/3785039179 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbd781a4a10 con 0x558ff7799c60 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a3af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd777fe700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c00a010 tx=0x7fbd6c0047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=75fb7e4acc54a3f2 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd577fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6c022070 con 0x558ff7799c60 2023-07-14T11:12:32.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558ff7645570 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.817+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fbd6c01dba0 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbd6c01dda0 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 10 client.25762 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x558ff7502ce0 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x558ff7645570 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fbd6c01c400 con 0x558ff7799c60 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 10 client.25762 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:32.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 10 client.25762 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 20 client.25762 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff77b4110 0x558ff77b6520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 4 client.25762 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.818+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558ff76e4f30 con 0x558ff77b4110 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.819+0000 7fbd77fff700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff77b4110 0x558ff77b6520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.819+0000 7fbd77fff700 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff77b4110 0x558ff77b6520 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=36c4b2917c3cce09 server_cookie=496eb414e0281efe in_seq=0 out_seq=0 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.819+0000 7fbd577fe700 10 client.25762 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:32.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd577fe700 1 -- 172.21.15.40:0/3785039179 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x558ff76e4f30 con 0x558ff77b4110 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd577fe700 10 client.25762 handle_command_reply: tid=0 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 client.25762 shutdown 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 msgr2=0x558ff77aa170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c00a010 tx=0x7fbd6c0047c0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 shutdown_connections 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff77b4110 0x558ff77b6520 crc :-1 s=READY pgs=52 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a3af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a9c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/3785039179 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77aa170 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd577fe700 0 client.25762 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 >> 172.21.15.40:0/3785039179 conn(0x558ff778fe20 msgr2=0x558ff7795310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 shutdown_connections 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 1 -- 172.21.15.40:0/3785039179 wait complete. 2023-07-14T11:12:32.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.820+0000 7fbd9217cb80 20 client.25762 trim_cache size 0 max 16384 2023-07-14T11:12:32.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 Processor -- start 2023-07-14T11:12:32.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 -- start start 2023-07-14T11:12:32.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a2b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a9d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff779aef0 con 0x558ff7799350 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff779a5c0 con 0x558ff77a4b50 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff778f9f0 con 0x558ff7799c60 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58652/0 (socket says 172.21.15.40:58652) 2023-07-14T11:12:32.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd777fe700 1 -- 172.21.15.40:0/2498134252 learned_addr learned my addr 172.21.15.40:0/2498134252 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd77fff700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a2b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.839+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a9d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 -- 172.21.15.40:0/2498134252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 msgr2=0x558ff77a9d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a9d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 -- 172.21.15.40:0/2498134252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 msgr2=0x558ff77a2b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a2b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 -- 172.21.15.40:0/2498134252 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558ff75402d0 con 0x558ff7799c60 2023-07-14T11:12:32.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd77fff700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a2b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:32.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd777fe700 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c009fe0 tx=0x7fbd6c00e430 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dfaf0098b5e7ed29 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd5effd700 1 -- 172.21.15.40:0/2498134252 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6c022070 con 0x558ff7799c60 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd5effd700 1 -- 172.21.15.40:0/2498134252 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd6c011ab0 con 0x558ff7799c60 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.840+0000 7fbd5effd700 1 -- 172.21.15.40:0/2498134252 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6c01ced0 con 0x558ff7799c60 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 -- 172.21.15.40:0/2498134252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 msgr2=0x558ff77a9800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c009fe0 tx=0x7fbd6c00e430 comp rx=0 tx=0).stop 2023-07-14T11:12:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 -- 172.21.15.40:0/2498134252 shutdown_connections 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff77a9d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7799350 0x558ff77a2b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/2498134252 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799c60 0x558ff77a9800 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 -- 172.21.15.40:0/2498134252 >> 172.21.15.40:0/2498134252 conn(0x558ff778fe20 msgr2=0x558ff77a3a10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 -- 172.21.15.40:0/2498134252 shutdown_connections 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.841+0000 7fbd9217cb80 1 -- 172.21.15.40:0/2498134252 wait complete. 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.842+0000 7fbd9217cb80 1 -- start start 2023-07-14T11:12:32.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.842+0000 7fbd9217cb80 1 Processor -- start 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.842+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799350 0x558ff780aa70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 0x558ff78051a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff78056e0 con 0x558ff7804d30 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff7805850 con 0x558ff77a4b50 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558ff7805a10 con 0x558ff7799350 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd9217cb80 10 client.0 _ll_register_callbacks cb 0x7fbd6c00e870 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:32.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59662/0 (socket says 172.21.15.40:59662) 2023-07-14T11:12:32.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 -- 172.21.15.40:0/1793742411 learned_addr learned my addr 172.21.15.40:0/1793742411 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:32.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd77fff700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799350 0x558ff780aa70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 0x558ff78051a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 -- 172.21.15.40:0/1793742411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799350 msgr2=0x558ff780aa70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799350 0x558ff780aa70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 -- 172.21.15.40:0/1793742411 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 msgr2=0x558ff78051a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:32.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 0x558ff78051a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd777fe700 1 -- 172.21.15.40:0/1793742411 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558ff7502ce0 con 0x558ff77a4b50 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.843+0000 7fbd7cd9c700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 0x558ff78051a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd777fe700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c016040 tx=0x7fbd6c01de50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4fd7ffba1c4d7243 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd577fe700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbd6c022040 con 0x558ff77a4b50 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fbd680a3680 con 0x558ff77a4b50 2023-07-14T11:12:32.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fbd6c01ce50 con 0x558ff77a4b50 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fbd6c005070 con 0x558ff77a4b50 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd9217cb80 10 client.25784 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x558ff7518330 con 0x558ff77a4b50 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.844+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fbd680a3680 con 0x558ff77a4b50 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fbd6c03c050 con 0x558ff77a4b50 2023-07-14T11:12:32.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 10 client.25784 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 10 client.25784 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 20 client.25784 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff7812950 0x558ff7814d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 4 client.25784 mds_command: new command op to 24569 tid=0[{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x558ff76a70d0 con 0x558ff7812950 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd77fff700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff7812950 0x558ff7814d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd77fff700 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff7812950 0x558ff7814d30 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8b293954d2432a36 server_cookie=e87a39fb3f03f427 in_seq=0 out_seq=0 2023-07-14T11:12:32.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.845+0000 7fbd577fe700 10 client.25784 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:32.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd577fe700 1 -- 172.21.15.40:0/1793742411 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x558ff76a70d0 con 0x558ff7812950 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd577fe700 10 client.25784 handle_command_reply: tid=0 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 client.25784 shutdown 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 msgr2=0x558ff780afb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7fbd6c016040 tx=0x7fbd6c01de50 comp rx=0 tx=0).stop 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 shutdown_connections 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558ff7812950 0x558ff7814d30 crc :-1 s=READY pgs=53 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd577fe700 0 client.25784 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558ff7804d30 0x558ff78051a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558ff7799350 0x558ff780aa70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 --2- 172.21.15.40:0/1793742411 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558ff77a4b50 0x558ff780afb0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.849+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 >> 172.21.15.40:0/1793742411 conn(0x558ff778fe20 msgr2=0x558ff77a2b30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.850+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 shutdown_connections 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.850+0000 7fbd9217cb80 1 -- 172.21.15.40:0/1793742411 wait complete. 2023-07-14T11:12:32.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.850+0000 7fbd9217cb80 20 client.25784 trim_cache size 0 max 16384 2023-07-14T11:12:32.857 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:12:32.857 INFO:teuthology.orchestra.run.smithi040.stdout: "return_code": 0, 2023-07-14T11:12:32.857 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_tag": "75928afe-1476-4940-a365-04191fbaa702", 2023-07-14T11:12:32.857 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": "asynchronous" 2023-07-14T11:12:32.857 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:12:32.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.851+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 msgr2=0x7fbd600a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.851+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 0x7fbd600a3880 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fbd781993d0 tx=0x7fbd6c009260 comp rx=0 tx=0).stop 2023-07-14T11:12:32.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.851+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 msgr2=0x7fbd78197dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:32.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.851+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 crypto rx=0x7fbd68000c00 tx=0x7fbd68005740 comp rx=0 tx=0).stop 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 shutdown_connections 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fbd600a13d0 0x7fbd600a3880 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fbd7810c310 0x7fbd7819c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbd780728b0 0x7fbd78197dc0 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 --2- 172.21.15.40:0/3556081626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fbd780731c0 0x7fbd78198300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 >> 172.21.15.40:0/3556081626 conn(0x7fbd780fb5f0 msgr2=0x7fbd7810e3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 shutdown_connections 2023-07-14T11:12:32.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:32.852+0000 7fbd7e7ff700 1 -- 172.21.15.40:0/3556081626 wait complete. 2023-07-14T11:12:32.877 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:12:32.972 INFO:teuthology.orchestra.run.smithi040.stderr:3100000 of 50000000 tuples (6%) done (elapsed 7.47 s, remaining 112.94 s) 2023-07-14T11:12:33.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 -- 172.21.15.40:0/708683932 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 msgr2=0x7f2ec01010d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/708683932 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec01010d0 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 crypto rx=0x7f2eb0009990 tx=0x7f2eb0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 -- 172.21.15.40:0/708683932 shutdown_connections 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/708683932 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 0x7f2ec010c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/708683932 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec01010d0 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/708683932 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec0109b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 -- 172.21.15.40:0/708683932 >> 172.21.15.40:0/708683932 conn(0x7f2ec00fa4b0 msgr2=0x7f2ec00fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 -- 172.21.15.40:0/708683932 shutdown_connections 2023-07-14T11:12:33.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.134+0000 7f2ec61ff700 1 -- 172.21.15.40:0/708683932 wait complete. 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.135+0000 7f2ec61ff700 1 Processor -- start 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.135+0000 7f2ec61ff700 1 -- start start 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec01982f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 0x7f2ec019c750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec019cd90 con 0x7f2ec0101610 2023-07-14T11:12:33.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec019cf00 con 0x7f2ec010a040 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ec61ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ec019d0c0 con 0x7f2ec0100d00 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebf7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec01982f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebf7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58664/0 (socket says 172.21.15.40:58664) 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebf7fe700 1 -- 172.21.15.40:0/1414171728 learned_addr learned my addr 172.21.15.40:0/1414171728 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebffff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 0x7f2ec019c750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 -- 172.21.15.40:0/1414171728 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 msgr2=0x7f2ec0197db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 -- 172.21.15.40:0/1414171728 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 msgr2=0x7f2ec019c750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 0x7f2ec019c750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebeffd700 1 -- 172.21.15.40:0/1414171728 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2eb00095f0 con 0x7f2ec0101610 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.136+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec01982f0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f2eb400e9f0 tx=0x7f2eb400ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ebcff9700 1 -- 172.21.15.40:0/1414171728 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb400c9c0 con 0x7f2ec0101610 2023-07-14T11:12:33.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2ec019d3a0 con 0x7f2ec0101610 2023-07-14T11:12:33.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ebcff9700 1 -- 172.21.15.40:0/1414171728 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2eb4010430 con 0x7f2ec0101610 2023-07-14T11:12:33.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ebcff9700 1 -- 172.21.15.40:0/1414171728 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb4018680 con 0x7f2ec0101610 2023-07-14T11:12:33.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.137+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ec01a4be0 con 0x7f2ec0101610 2023-07-14T11:12:33.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.142+0000 7f2ebcff9700 1 -- 172.21.15.40:0/1414171728 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f2eb4010860 con 0x7f2ec0101610 2023-07-14T11:12:33.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 Processor -- start 2023-07-14T11:12:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 -- start start 2023-07-14T11:12:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa6eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.143+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1e9fcb0 con 0x560ef1f99be0 2023-07-14T11:12:33.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1ef2a40 con 0x560ef1fa4ad0 2023-07-14T11:12:33.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1eebed0 con 0x560ef1f992d0 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58670/0 (socket says 172.21.15.40:58670) 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 -- 172.21.15.40:0/785618616 learned_addr learned my addr 172.21.15.40:0/785618616 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebcff9700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 0x7f2ea80a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa6eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 -- 172.21.15.40:0/785618616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 msgr2=0x560ef1fa6eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa6eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 -- 172.21.15.40:0/785618616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 msgr2=0x560ef1fa4590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebcff9700 1 -- 172.21.15.40:0/1414171728 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2eb4014070 con 0x7f2ec0101610 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 -- 172.21.15.40:0/785618616 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ef1d19de0 con 0x560ef1f992d0 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.144+0000 7f2ebffff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 0x7f2ea80a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.145+0000 7f2ebffff700 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8011790 tx=0x7f2eb8011aa0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=914a673f125975c9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.145+0000 7f2ea5ffb700 1 -- 172.21.15.40:0/785618616 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb800e040 con 0x560ef1f992d0 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.145+0000 7f2ea5ffb700 1 -- 172.21.15.40:0/785618616 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2eb8027070 con 0x560ef1f992d0 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.145+0000 7f2ea5ffb700 1 -- 172.21.15.40:0/785618616 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb80237d0 con 0x560ef1f992d0 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/785618616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 msgr2=0x560ef1f996a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ebffff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 0x7f2ea80a3800 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8022000 tx=0x7f2eb801f000 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8011790 tx=0x7f2eb8011aa0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/785618616 shutdown_connections 2023-07-14T11:12:33.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa6eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1f996a0 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/785618616 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/785618616 >> 172.21.15.40:0/785618616 conn(0x560ef1f8fda0 msgr2=0x560ef1f956a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/785618616 shutdown_connections 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/785618616 wait complete. 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.146+0000 7f2ed9a32b80 1 -- start start 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 Processor -- start 2023-07-14T11:12:33.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1f992d0 0x560ef1fa4520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa1ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1e8bac0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1e8b7c0 con 0x560ef1f992d0 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1d6f920 con 0x560ef1f99be0 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ed9a32b80 10 client.0 _ll_register_callbacks cb 0x560ef1e89760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ebeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ebeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58676/0 (socket says 172.21.15.40:58676) 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa1ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.147+0000 7f2ebeffd700 1 -- 172.21.15.40:0/3526655418 learned_addr learned my addr 172.21.15.40:0/3526655418 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1f992d0 0x560ef1fa4520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 -- 172.21.15.40:0/3526655418 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 msgr2=0x560ef1fa17a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 -- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1f992d0 msgr2=0x560ef1fa4520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1f992d0 0x560ef1fa4520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 -- 172.21.15.40:0/3526655418 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2eb8011380 con 0x560ef1fa4ad0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2ebffff700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa1ee0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8011790 tx=0x7f2eb8009620 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ef6e5b562ea73097 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.148+0000 7f2e9effd700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb8023c70 con 0x560ef1fa4ad0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560ef1e454f0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2eb8023dd0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2eb8004a50 con 0x560ef1fa4ad0 2023-07-14T11:12:33.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2ed9a32b80 10 client.15792 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:33.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x560ef1d02c60 con 0x560ef1fa4ad0 2023-07-14T11:12:33.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560ef1e454f0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.149+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2eb8018c10 con 0x560ef1fa4ad0 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 10 client.15792 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 10 client.15792 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 20 client.15792 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef1fb4440 0x560ef1fb6850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 4 client.15792 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560ef1ee4eb0 con 0x560ef1fb4440 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef1fb4440 0x560ef1fb6850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef1fb4440 0x560ef1fb6850 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3208b877eceb714f server_cookie=c49844953f1f6c37 in_seq=0 out_seq=0 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.150+0000 7f2e9effd700 10 client.15792 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2e9effd700 1 -- 172.21.15.40:0/3526655418 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x560ef1ee4eb0 con 0x560ef1fb4440 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2e9effd700 10 client.15792 handle_command_reply: tid=0 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 client.15792 shutdown 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 msgr2=0x560ef1fa1ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa1ee0 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8011790 tx=0x7f2eb8009620 comp rx=0 tx=0).stop 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 shutdown_connections 2023-07-14T11:12:33.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef1fb4440 0x560ef1fb6850 crc :-1 s=READY pgs=54 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1fa4ad0 0x560ef1fa1ee0 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1f992d0 0x560ef1fa4520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/3526655418 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1fa17a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 >> 172.21.15.40:0/3526655418 conn(0x560ef1f8fda0 msgr2=0x560ef1f95230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.151+0000 7f2e9effd700 0 client.15792 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.152+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 shutdown_connections 2023-07-14T11:12:33.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.152+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/3526655418 wait complete. 2023-07-14T11:12:33.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.152+0000 7f2ed9a32b80 20 client.15792 trim_cache size 0 max 16384 2023-07-14T11:12:33.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 Processor -- start 2023-07-14T11:12:33.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 -- start start 2023-07-14T11:12:33.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1fa0d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1f9ea50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1e454f0 con 0x560ef1f99be0 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1f9ae30 con 0x560ef1fa4ad0 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.175+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef1f96d00 con 0x560ef1f992d0 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59218/0 (socket says 172.21.15.40:59218) 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1f9ea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 -- 172.21.15.40:0/165551751 learned_addr learned my addr 172.21.15.40:0/165551751 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1fa0d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 -- 172.21.15.40:0/165551751 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 msgr2=0x560ef1fa0d30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1fa0d30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 -- 172.21.15.40:0/165551751 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 msgr2=0x560ef1f9ea50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1f9ea50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 -- 172.21.15.40:0/165551751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ef1d40250 con 0x560ef1f99be0 2023-07-14T11:12:33.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1fa0d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebffff700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1f9ea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.176+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f2eb4000c00 tx=0x7f2eb40cf040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f01fd305323256e3 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.177+0000 7f2ea67fc700 1 -- 172.21.15.40:0/165551751 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb4099a80 con 0x560ef1f99be0 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.177+0000 7f2ea67fc700 1 -- 172.21.15.40:0/165551751 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2eb40998a0 con 0x560ef1f99be0 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.177+0000 7f2ea67fc700 1 -- 172.21.15.40:0/165551751 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb40a1f00 con 0x560ef1f99be0 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.177+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/165551751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 msgr2=0x560ef1fa4050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.177+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 crypto rx=0x7f2eb4000c00 tx=0x7f2eb40cf040 comp rx=0 tx=0).stop 2023-07-14T11:12:33.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/165551751 shutdown_connections 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1f9ea50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f992d0 0x560ef1fa0d30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/165551751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f99be0 0x560ef1fa4050 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/165551751 >> 172.21.15.40:0/165551751 conn(0x560ef1f8fda0 msgr2=0x560ef1fb0760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/165551751 shutdown_connections 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/165551751 wait complete. 2023-07-14T11:12:33.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.178+0000 7f2ed9a32b80 1 -- start start 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 Processor -- start 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 0x560ef1fa9b20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1faa060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef2008310 con 0x560ef1f992d0 2023-07-14T11:12:33.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef2008450 con 0x560ef1fa4ad0 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ef2008590 con 0x560ef1f99be0 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.179+0000 7f2ed9a32b80 10 client.0 _ll_register_callbacks cb 0x7f2eb4005ec0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebf7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 0x560ef1fa9b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:59702/0 (socket says 172.21.15.40:59702) 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 -- 172.21.15.40:0/2955539460 learned_addr learned my addr 172.21.15.40:0/2955539460 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebeffd700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1faa060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 -- 172.21.15.40:0/2955539460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 msgr2=0x560ef1faa060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1faa060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 -- 172.21.15.40:0/2955539460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 msgr2=0x560ef1fa9b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 0x560ef1fa9b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 -- 172.21.15.40:0/2955539460 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ec010d170 con 0x560ef1fa4ad0 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 0x560ef1fa9b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2ebffff700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8016a70 tx=0x7f2eb8007e40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4273ebe4873a2d5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.180+0000 7f2e9effd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2eb800e030 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2eb8027070 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2eb80052e0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2eb80060a0 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 10 client.25802 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x560ef1d02c60 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2eb8027070 con 0x560ef1fa4ad0 2023-07-14T11:12:33.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2eb8004a50 con 0x560ef1fa4ad0 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 10 client.25802 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 10 client.25802 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 20 client.25802 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef20099e0 0x560ef200bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 4 client.25802 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.181+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x560ef1ea7050 con 0x560ef20099e0 2023-07-14T11:12:33.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef20099e0 0x560ef200bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2ebf7fe700 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef20099e0 0x560ef200bdf0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=394b8b9a3847283b server_cookie=91cbed25ff4f6014 in_seq=0 out_seq=0 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2e9effd700 10 client.25802 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2e9effd700 1 -- 172.21.15.40:0/2955539460 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x560ef1ea7050 con 0x560ef20099e0 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2e9effd700 10 client.25802 handle_command_reply: tid=0 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.182+0000 7f2ed9a32b80 1 client.25802 shutdown 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 msgr2=0x560ef1faa5a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8016a70 tx=0x7f2eb8007e40 comp rx=0 tx=0).stop 2023-07-14T11:12:33.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 shutdown_connections 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x560ef20099e0 0x560ef200bdf0 crc :-1 s=READY pgs=55 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ef1fa4ad0 0x560ef1faa5a0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ef1f992d0 0x560ef1fa9b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2e9effd700 0 client.25802 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 --2- 172.21.15.40:0/2955539460 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ef1f99be0 0x560ef1faa060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 >> 172.21.15.40:0/2955539460 conn(0x560ef1f8fda0 msgr2=0x560ef1f9b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 shutdown_connections 2023-07-14T11:12:33.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 1 -- 172.21.15.40:0/2955539460 wait complete. 2023-07-14T11:12:33.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.183+0000 7f2ed9a32b80 20 client.25802 trim_cache size 0 max 16384 2023-07-14T11:12:33.193 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:12:33.193 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "no active scrubs running", 2023-07-14T11:12:33.193 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": {} 2023-07-14T11:12:33.193 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:12:33.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 msgr2=0x7f2ea80a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 0x7f2ea80a3800 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f2eb8022000 tx=0x7f2eb801f000 comp rx=0 tx=0).stop 2023-07-14T11:12:33.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 msgr2=0x7f2ec01982f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec01982f0 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 crypto rx=0x7f2eb400e9f0 tx=0x7f2eb400ed00 comp rx=0 tx=0).stop 2023-07-14T11:12:33.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 shutdown_connections 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2ea80a1350 0x7f2ea80a3800 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2ec010a040 0x7f2ec019c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2ec0100d00 0x7f2ec0197db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 --2- 172.21.15.40:0/1414171728 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2ec0101610 0x7f2ec01982f0 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 >> 172.21.15.40:0/1414171728 conn(0x7f2ec00fa4b0 msgr2=0x7f2ec010c190 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 shutdown_connections 2023-07-14T11:12:33.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.184+0000 7f2ec61ff700 1 -- 172.21.15.40:0/1414171728 wait complete. 2023-07-14T11:12:33.212 INFO:tasks.cephfs.filesystem:all active scrubs completed 2023-07-14T11:12:33.212 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:12:33.286 INFO:teuthology.orchestra.run.smithi040.stderr:3200000 of 50000000 tuples (6%) done (elapsed 7.78 s, remaining 113.77 s) 2023-07-14T11:12:33.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/1644304315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 msgr2=0x7fcb041023b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/1644304315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb041023b0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 crypto rx=0x7fcb00009920 tx=0x7fcb00009c30 comp rx=0 tx=0).stop 2023-07-14T11:12:33.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/1644304315 shutdown_connections 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/1644304315 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 0x7fcb0410ebf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/1644304315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb041023b0 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/1644304315 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb04106940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/1644304315 >> 172.21.15.40:0/1644304315 conn(0x7fcb040fd870 msgr2=0x7fcb040ffc80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.429+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/1644304315 shutdown_connections 2023-07-14T11:12:33.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.430+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/1644304315 wait complete. 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.430+0000 7fcb0c7ff700 1 Processor -- start 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 -- start start 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb041983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 0x7fcb0419c840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0419ce80 con 0x7fcb04101fe0 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0419cff0 con 0x7fcb04107020 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0c7ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb0419d1b0 con 0x7fcb04102980 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0a59b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb09d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb041983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0a59b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59234/0 (socket says 172.21.15.40:59234) 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0a59b700 1 -- 172.21.15.40:0/405072273 learned_addr learned my addr 172.21.15.40:0/405072273 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.431+0000 7fcb0ad9c700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 0x7fcb0419c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 -- 172.21.15.40:0/405072273 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 msgr2=0x7fcb0419c840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 0x7fcb0419c840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 -- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 msgr2=0x7fcb04197ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 -- 172.21.15.40:0/405072273 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcb00009520 con 0x7fcb04102980 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb0a59b700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb09d9a700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb041983e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fcb04102d50 tx=0x7fcaf400eca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf4010900 con 0x7fcb04102980 2023-07-14T11:12:33.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcb0419d490 con 0x7fcb04102980 2023-07-14T11:12:33.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcaf4018420 con 0x7fcb04102980 2023-07-14T11:12:33.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.432+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcaf400f680 con 0x7fcb04102980 2023-07-14T11:12:33.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.433+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcb041a4cd0 con 0x7fcb04102980 2023-07-14T11:12:33.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.434+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcb04067750 con 0x7fcb04102980 2023-07-14T11:12:33.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.436+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7fcaf4018590 con 0x7fcb04102980 2023-07-14T11:12:33.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.437+0000 7fcafb7fe700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 0x7fcaf00a3840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.438+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fcaf4014070 con 0x7fcb04102980 2023-07-14T11:12:33.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.438+0000 7fcb0a59b700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 0x7fcaf00a3840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.438+0000 7fcb0a59b700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 0x7fcaf00a3840 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fcb0000c010 tx=0x7fcb00019040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.439+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fcaf4096130 con 0x7fcb04102980 2023-07-14T11:12:33.619 INFO:teuthology.orchestra.run.smithi040.stderr:3300000 of 50000000 tuples (6%) done (elapsed 8.11 s, remaining 114.79 s) 2023-07-14T11:12:33.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.730+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fcb041991f0 con 0x7fcb04102980 2023-07-14T11:12:33.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.731+0000 7fcafb7fe700 1 -- 172.21.15.40:0/405072273 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 27 v27) v1 ==== 94+0+10890 (secure 0 0 0) 0x7fcaf4099750 con 0x7fcb04102980 2023-07-14T11:12:33.734 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:12:33.734 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":27,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":27,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:10:17.374269+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:12:33.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 msgr2=0x7fcaf00a3840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 0x7fcaf00a3840 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fcb0000c010 tx=0x7fcb00019040 comp rx=0 tx=0).stop 2023-07-14T11:12:33.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 msgr2=0x7fcb041983e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb041983e0 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 crypto rx=0x7fcb04102d50 tx=0x7fcaf400eca0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 shutdown_connections 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcaf00a1390 0x7fcaf00a3840 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcb04107020 0x7fcb0419c840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb04101fe0 0x7fcb04197ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 --2- 172.21.15.40:0/405072273 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcb04102980 0x7fcb041983e0 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.733+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 >> 172.21.15.40:0/405072273 conn(0x7fcb040fd870 msgr2=0x7fcb040fe4b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.734+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 shutdown_connections 2023-07-14T11:12:33.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.734+0000 7fcb0c7ff700 1 -- 172.21.15.40:0/405072273 wait complete. 2023-07-14T11:12:33.738 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 27 2023-07-14T11:12:33.756 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:4 damage ls 2023-07-14T11:12:33.829 INFO:teuthology.orchestra.run.smithi040.stderr:3400000 of 50000000 tuples (6%) done (elapsed 8.32 s, remaining 114.06 s) 2023-07-14T11:12:33.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 -- 172.21.15.40:0/3526471638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4105af0 msgr2=0x7f11b4105ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/3526471638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4105af0 0x7f11b4105ec0 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 crypto rx=0x7f11a4009990 tx=0x7f11a4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 -- 172.21.15.40:0/3526471638 shutdown_connections 2023-07-14T11:12:33.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/3526471638 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b4100490 0x7f11b410c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/3526471638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4105af0 0x7f11b4105ec0 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/3526471638 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b40ffae0 0x7f11b40fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 -- 172.21.15.40:0/3526471638 >> 172.21.15.40:0/3526471638 conn(0x7f11b40fb5e0 msgr2=0x7f11b40fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:33.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 -- 172.21.15.40:0/3526471638 shutdown_connections 2023-07-14T11:12:33.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.985+0000 7f11ba3ff700 1 -- 172.21.15.40:0/3526471638 wait complete. 2023-07-14T11:12:33.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.986+0000 7f11ba3ff700 1 Processor -- start 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.986+0000 7f11ba3ff700 1 -- start start 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4100490 0x7f11b419bef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 0x7f11b4195a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11b4196090 con 0x7f11b4100490 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11b4196200 con 0x7f11b40ffae0 2023-07-14T11:12:33.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11ba3ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11b41963c0 con 0x7f11b4105af0 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60390/0 (socket says 172.21.15.40:60390) 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 -- 172.21.15.40:0/1474426540 learned_addr learned my addr 172.21.15.40:0/1474426540 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b37fe700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4100490 0x7f11b419bef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b899c700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 0x7f11b4195a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 -- 172.21.15.40:0/1474426540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 msgr2=0x7f11b4195a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 0x7f11b4195a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 -- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4100490 msgr2=0x7f11b419bef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4100490 0x7f11b419bef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b3fff700 1 -- 172.21.15.40:0/1474426540 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11a40095f0 con 0x7f11b40ffae0 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.987+0000 7f11b899c700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 0x7f11b4195a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11b3fff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f11a4005910 tx=0x7f11a4004710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11b17fa700 1 -- 172.21.15.40:0/1474426540 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a401d070 con 0x7f11b40ffae0 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11b4196640 con 0x7f11b40ffae0 2023-07-14T11:12:33.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11b17fa700 1 -- 172.21.15.40:0/1474426540 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11a4022470 con 0x7f11b40ffae0 2023-07-14T11:12:33.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11b17fa700 1 -- 172.21.15.40:0/1474426540 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a4017650 con 0x7f11b40ffae0 2023-07-14T11:12:33.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.988+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11b41a29a0 con 0x7f11b40ffae0 2023-07-14T11:12:33.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.991+0000 7f11b17fa700 1 -- 172.21.15.40:0/1474426540 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f11a40225e0 con 0x7f11b40ffae0 2023-07-14T11:12:33.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.993+0000 7f11cdc7bb80 1 Processor -- start 2023-07-14T11:12:33.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.993+0000 7f11b17fa700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 0x7f119c0a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.993+0000 7f11cdc7bb80 1 -- start start 2023-07-14T11:12:33.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.993+0000 7f11b17fa700 1 -- 172.21.15.40:0/1474426540 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f11a40ce4d0 con 0x7f11b40ffae0 2023-07-14T11:12:33.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.993+0000 7f11b899c700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 0x7f119c0a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a7779a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a777ee0 0x55cf4a77a2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a66ecb0 con 0x55cf4a777ee0 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a6c1a40 con 0x55cf4a76c6e0 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a6baed0 con 0x55cf4a76cff0 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60406/0 (socket says 172.21.15.40:60406) 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 -- 172.21.15.40:0/3274878262 learned_addr learned my addr 172.21.15.40:0/3274878262 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b899c700 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a777ee0 0x55cf4a77a2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b37fe700 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a7779a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b899c700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 0x7f119c0a3800 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f11b419c6b0 tx=0x7f11ac009920 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 -- 172.21.15.40:0/3274878262 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 msgr2=0x55cf4a7779a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a7779a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 -- 172.21.15.40:0/3274878262 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a777ee0 msgr2=0x55cf4a77a2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:33.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a777ee0 0x55cf4a77a2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.994+0000 7f11b3fff700 1 -- 172.21.15.40:0/3274878262 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf4a4e8de0 con 0x55cf4a76c6e0 2023-07-14T11:12:34.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.995+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f11a40cb880 tx=0x7f11a409aff0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=dd53472c1781cbe1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.995+0000 7f119a7fc700 1 -- 172.21.15.40:0/3274878262 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a409b150 con 0x55cf4a76c6e0 2023-07-14T11:12:34.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.995+0000 7f119a7fc700 1 -- 172.21.15.40:0/3274878262 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11a409f180 con 0x55cf4a76c6e0 2023-07-14T11:12:34.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.995+0000 7f119a7fc700 1 -- 172.21.15.40:0/3274878262 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a409b610 con 0x55cf4a76c6e0 2023-07-14T11:12:34.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3274878262 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 msgr2=0x55cf4a76cab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f11a40cb880 tx=0x7f11a409aff0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3274878262 shutdown_connections 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a777ee0 0x55cf4a77a2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76cab0 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3274878262 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a7779a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3274878262 >> 172.21.15.40:0/3274878262 conn(0x55cf4a75eda0 msgr2=0x55cf4a7646a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3274878262 shutdown_connections 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3274878262 wait complete. 2023-07-14T11:12:34.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.996+0000 7f11cdc7bb80 1 -- start start 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.997+0000 7f11cdc7bb80 1 Processor -- start 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 0x55cf4a77cf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a77d440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 0x55cf4a776df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a65aac0 con 0x55cf4a76c6e0 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a65a7c0 con 0x55cf4a777ee0 2023-07-14T11:12:34.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a53e920 con 0x55cf4a76cff0 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11cdc7bb80 10 client.0 _ll_register_callbacks cb 0x55cf4a658760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11b37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a77d440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11b37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a77d440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49242/0 (socket says 172.21.15.40:49242) 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11b37fe700 1 -- 172.21.15.40:0/2813853615 learned_addr learned my addr 172.21.15.40:0/2813853615 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11b899c700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 0x55cf4a776df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.998+0000 7f11b3fff700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 0x55cf4a77cf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 -- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 msgr2=0x55cf4a77d440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a77d440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 -- 172.21.15.40:0/2813853615 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 msgr2=0x55cf4a776df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 0x55cf4a776df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 -- 172.21.15.40:0/2813853615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11b41a29a0 con 0x55cf4a76c6e0 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b899c700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 0x55cf4a776df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11b3fff700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 0x55cf4a77cf00 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f11a40994a0 tx=0x7f11a4022b30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=d94ed76b2909b6f8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11937fe700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:34.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a409b150 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f11a409a850 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55cf4a6144f0 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:33.999+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f11a40b9250 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 10 client.15816 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55cf4a4d1c60 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11a40cb020 con 0x55cf4a76c6e0 2023-07-14T11:12:34.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f11a40a3080 con 0x55cf4a76c6e0 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 10 client.15816 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 10 client.15816 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 20 client.15816 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a786ed0 0x55cf4a7892e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 4 client.15816 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55cf4a6b3eb0 con 0x55cf4a786ed0 2023-07-14T11:12:34.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.000+0000 7f11b37fe700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a786ed0 0x55cf4a7892e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.001+0000 7f11b37fe700 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a786ed0 0x55cf4a7892e0 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=b1c555af5de656c5 server_cookie=43d2d2b97cc7224a in_seq=0 out_seq=0 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.001+0000 7f11937fe700 10 client.15816 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.001+0000 7f11937fe700 1 -- 172.21.15.40:0/2813853615 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55cf4a6b3eb0 con 0x55cf4a786ed0 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11937fe700 10 client.15816 handle_command_reply: tid=0 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 client.15816 shutdown 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 msgr2=0x55cf4a77cf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 0x55cf4a77cf00 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 crypto rx=0x7f11a40994a0 tx=0x7f11a4022b30 comp rx=0 tx=0).stop 2023-07-14T11:12:34.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 shutdown_connections 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a786ed0 0x55cf4a7892e0 crc :-1 s=READY pgs=32 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a777ee0 0x55cf4a776df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76c6e0 0x55cf4a77cf00 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11937fe700 0 client.15816 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/2813853615 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a77d440 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 >> 172.21.15.40:0/2813853615 conn(0x55cf4a75eda0 msgr2=0x55cf4a764230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 shutdown_connections 2023-07-14T11:12:34.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/2813853615 wait complete. 2023-07-14T11:12:34.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.002+0000 7f11cdc7bb80 20 client.15816 trim_cache size 0 max 16384 2023-07-14T11:12:34.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.016+0000 7f11cdc7bb80 1 Processor -- start 2023-07-14T11:12:34.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.016+0000 7f11cdc7bb80 1 -- start start 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76ecc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a76f200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a76e280 con 0x55cf4a75eda0 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a6b3eb0 con 0x55cf4a76c6e0 2023-07-14T11:12:34.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a76d9a0 con 0x55cf4a76cff0 2023-07-14T11:12:34.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39256/0 (socket says 172.21.15.40:39256) 2023-07-14T11:12:34.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11b3fff700 1 -- 172.21.15.40:0/4196799478 learned_addr learned my addr 172.21.15.40:0/4196799478 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11b37fe700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76ecc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.017+0000 7f11b899c700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a76f200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 -- 172.21.15.40:0/4196799478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 msgr2=0x55cf4a76f200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a76f200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 -- 172.21.15.40:0/4196799478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 msgr2=0x55cf4a76ecc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76ecc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 -- 172.21.15.40:0/4196799478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55cf4a50f250 con 0x55cf4a75eda0 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b37fe700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76ecc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f11b3fff700 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f11a40ce480 tx=0x7f11a409aff0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3345ff23a3cba2a7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f119affd700 1 -- 172.21.15.40:0/4196799478 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a40cbd80 con 0x55cf4a75eda0 2023-07-14T11:12:34.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f119affd700 1 -- 172.21.15.40:0/4196799478 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11a40cb040 con 0x55cf4a75eda0 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.018+0000 7f119affd700 1 -- 172.21.15.40:0/4196799478 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a40cb280 con 0x55cf4a75eda0 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/4196799478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 msgr2=0x55cf4a776f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 crypto rx=0x7f11a40ce480 tx=0x7f11a409aff0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/4196799478 shutdown_connections 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a76cff0 0x55cf4a76f200 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a75eda0 0x55cf4a776f20 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/4196799478 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a76ecc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/4196799478 >> 172.21.15.40:0/4196799478 conn(0x55cf4a760690 msgr2=0x55cf4a76e5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/4196799478 shutdown_connections 2023-07-14T11:12:34.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.019+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/4196799478 wait complete. 2023-07-14T11:12:34.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 -- start start 2023-07-14T11:12:34.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 Processor -- start 2023-07-14T11:12:34.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a7cf450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76cff0 0x55cf4a7d9da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a7da2e0 con 0x55cf4a76cff0 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a7da420 con 0x55cf4a76c6e0 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55cf4a7da560 con 0x55cf4a75eda0 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.020+0000 7f11cdc7bb80 10 client.0 _ll_register_callbacks cb 0x7f11ac009e80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49270/0 (socket says 172.21.15.40:49270) 2023-07-14T11:12:34.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 -- 172.21.15.40:0/3163028689 learned_addr learned my addr 172.21.15.40:0/3163028689 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b899c700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76cff0 0x55cf4a7d9da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 -- 172.21.15.40:0/3163028689 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 msgr2=0x55cf4a7cf450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a7cf450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 -- 172.21.15.40:0/3163028689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76cff0 msgr2=0x55cf4a7d9da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76cff0 0x55cf4a7d9da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.021+0000 7f11b3fff700 1 -- 172.21.15.40:0/3163028689 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55cf4a4d1c60 con 0x55cf4a75eda0 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11b3fff700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f11a4097bc0 tx=0x7f11a409ac90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6e0b55759c4012a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11937fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f11a40b9450 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f11a40a3510 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f11a409f180 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f11a409b3b0 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11cdc7bb80 10 client.25810 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.022+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55cf4a4e72b0 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f11a409f180 con 0x55cf4a75eda0 2023-07-14T11:12:34.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f11a4096f40 con 0x55cf4a75eda0 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 10 client.25810 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 10 client.25810 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 20 client.25810 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a777ee0 0x55cf4a7dd6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 4 client.25810 mds_command: new command op to 24511 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55cf4a676050 con 0x55cf4a777ee0 2023-07-14T11:12:34.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.023+0000 7f11b37fe700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a777ee0 0x55cf4a7dd6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11937fe700 10 client.25810 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11b37fe700 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a777ee0 0x55cf4a7dd6e0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=1f7dd79d7fb64dfb server_cookie=a928bfb8bfe75b2 in_seq=0 out_seq=0 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11937fe700 1 -- 172.21.15.40:0/3163028689 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55cf4a676050 con 0x55cf4a777ee0 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11937fe700 10 client.25810 handle_command_reply: tid=0 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 client.25810 shutdown 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 msgr2=0x55cf4a7cef10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 crypto rx=0x7f11a4097bc0 tx=0x7f11a409ac90 comp rx=0 tx=0).stop 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 shutdown_connections 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55cf4a777ee0 0x55cf4a7dd6e0 crc :-1 s=READY pgs=33 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55cf4a76cff0 0x55cf4a7d9da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55cf4a75eda0 0x55cf4a7cef10 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 --2- 172.21.15.40:0/3163028689 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55cf4a76c6e0 0x55cf4a7cf450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 >> 172.21.15.40:0/3163028689 conn(0x55cf4a760690 msgr2=0x55cf4a7638b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.024+0000 7f11937fe700 0 client.25810 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:12:34.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 shutdown_connections 2023-07-14T11:12:34.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11cdc7bb80 1 -- 172.21.15.40:0/3163028689 wait complete. 2023-07-14T11:12:34.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11cdc7bb80 20 client.25810 trim_cache size 0 max 16384 2023-07-14T11:12:34.034 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 msgr2=0x7f119c0a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 0x7f119c0a3800 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f11b419c6b0 tx=0x7f11ac009920 comp rx=0 tx=0).stop 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.025+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 msgr2=0x7f11b419b9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f11a4005910 tx=0x7f11a4004710 comp rx=0 tx=0).stop 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 shutdown_connections 2023-07-14T11:12:34.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f119c0a1350 0x7f119c0a3800 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f11b4105af0 0x7f11b4195a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f11b40ffae0 0x7f11b419b9b0 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 --2- 172.21.15.40:0/1474426540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11b4100490 0x7f11b419bef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 >> 172.21.15.40:0/1474426540 conn(0x7f11b40fb5e0 msgr2=0x7f11b40fbc40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 shutdown_connections 2023-07-14T11:12:34.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.026+0000 7f11ba3ff700 1 -- 172.21.15.40:0/1474426540 wait complete. 2023-07-14T11:12:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: pgmap v15: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 31 MiB/s wr, 145 op/s 2023-07-14T11:12:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: scrub queued for path: / 2023-07-14T11:12:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:12:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:33 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:12:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: pgmap v15: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 31 MiB/s wr, 145 op/s 2023-07-14T11:12:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: scrub queued for path: / 2023-07-14T11:12:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:12:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:33 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:12:34.055 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:3 damage ls 2023-07-14T11:12:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: pgmap v15: 129 pgs: 129 active+clean; 219 MiB data, 742 MiB used, 1.0 TiB / 1.0 TiB avail; 204 KiB/s rd, 31 MiB/s wr, 145 op/s 2023-07-14T11:12:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: scrub queued for path: / 2023-07-14T11:12:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:12:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:33 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:12:34.122 INFO:teuthology.orchestra.run.smithi040.stderr:3500000 of 50000000 tuples (7%) done (elapsed 8.61 s, remaining 114.45 s) 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 -- 172.21.15.40:0/3880600681 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c1013b0 msgr2=0x7f890c109a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 --2- 172.21.15.40:0/3880600681 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c1013b0 0x7f890c109a80 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f88f8009990 tx=0x7f88f8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 -- 172.21.15.40:0/3880600681 shutdown_connections 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 --2- 172.21.15.40:0/3880600681 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c109fc0 0x7f890c10c3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 --2- 172.21.15.40:0/3880600681 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c100aa0 0x7f890c100e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 --2- 172.21.15.40:0/3880600681 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c1013b0 0x7f890c109a80 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 -- 172.21.15.40:0/3880600681 >> 172.21.15.40:0/3880600681 conn(0x7f890c0745d0 msgr2=0x7f890c0749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 -- 172.21.15.40:0/3880600681 shutdown_connections 2023-07-14T11:12:34.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.297+0000 7f8911f9d700 1 -- 172.21.15.40:0/3880600681 wait complete. 2023-07-14T11:12:34.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.299+0000 7f8911f9d700 1 Processor -- start 2023-07-14T11:12:34.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.299+0000 7f8911f9d700 1 -- start start 2023-07-14T11:12:34.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 0x7f890c199aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 0x7f890c193640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890c193ca0 con 0x7f890c1013b0 2023-07-14T11:12:34.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890c193e10 con 0x7f890c100aa0 2023-07-14T11:12:34.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8911f9d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890c193fd0 con 0x7f890c109fc0 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60434/0 (socket says 172.21.15.40:60434) 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 -- 172.21.15.40:0/1166492768 learned_addr learned my addr 172.21.15.40:0/1166492768 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f891179c700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 0x7f890c193640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f890bfff700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 0x7f890c199aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 -- 172.21.15.40:0/1166492768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 msgr2=0x7f890c193640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 0x7f890c193640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 -- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 msgr2=0x7f890c199aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 0x7f890c199aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f8910f9b700 1 -- 172.21.15.40:0/1166492768 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88f80095f0 con 0x7f890c100aa0 2023-07-14T11:12:34.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f891179c700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 0x7f890c193640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:34.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.300+0000 7f890bfff700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 0x7f890c199aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f890000bc50 tx=0x7f8900008510 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8909ffb700 1 -- 172.21.15.40:0/1166492768 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8900008b20 con 0x7f890c100aa0 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f890c1942b0 con 0x7f890c100aa0 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8909ffb700 1 -- 172.21.15.40:0/1166492768 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8900008c80 con 0x7f890c100aa0 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8909ffb700 1 -- 172.21.15.40:0/1166492768 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8900015640 con 0x7f890c100aa0 2023-07-14T11:12:34.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.301+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f890c1a0550 con 0x7f890c100aa0 2023-07-14T11:12:34.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.307+0000 7f8909ffb700 1 -- 172.21.15.40:0/1166492768 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f8900016480 con 0x7f890c100aa0 2023-07-14T11:12:34.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.308+0000 7f8909ffb700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 0x7f88fc0b4930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.309+0000 7f8926bc1b80 1 Processor -- start 2023-07-14T11:12:34.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.309+0000 7f8909ffb700 1 -- 172.21.15.40:0/1166492768 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f89000cd080 con 0x7f890c100aa0 2023-07-14T11:12:34.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.309+0000 7f8926bc1b80 1 -- start start 2023-07-14T11:12:34.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.309+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c577f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5a110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9b52cb0 con 0x55b8a9c57d30 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9ba5a40 con 0x55b8a9c4de00 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9b9eed0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f890bfff700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 0x7f88fc0b4930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49298/0 (socket says 172.21.15.40:49298) 2023-07-14T11:12:34.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f8910f9b700 1 -- 172.21.15.40:0/4133806177 learned_addr learned my addr 172.21.15.40:0/4133806177 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f890bfff700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c577f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.310+0000 7f891179c700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5a110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 -- 172.21.15.40:0/4133806177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 msgr2=0x55b8a9c4d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 -- 172.21.15.40:0/4133806177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 msgr2=0x55b8a9c5a110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5a110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 -- 172.21.15.40:0/4133806177 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b8a99ccde0 con 0x55b8a9c4de00 2023-07-14T11:12:34.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f8910f9b700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c577f0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f88f8006010 tx=0x7f88f800bcd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3a6fbf9293561716 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f890bfff700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 0x7f88fc0b4930 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f88f800b650 tx=0x7f88f801b000 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.311+0000 7f88f6ffd700 1 -- 172.21.15.40:0/4133806177 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f88f8005c00 con 0x55b8a9c4de00 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.312+0000 7f88f6ffd700 1 -- 172.21.15.40:0/4133806177 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f88f80054f0 con 0x55b8a9c4de00 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.312+0000 7f88f6ffd700 1 -- 172.21.15.40:0/4133806177 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f88f8004d30 con 0x55b8a9c4de00 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.312+0000 7f8926bc1b80 1 -- 172.21.15.40:0/4133806177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 msgr2=0x55b8a9c577f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.312+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c577f0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f88f8006010 tx=0x7f88f800bcd0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 -- 172.21.15.40:0/4133806177 shutdown_connections 2023-07-14T11:12:34.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5a110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c4d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/4133806177 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c577f0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 -- 172.21.15.40:0/4133806177 >> 172.21.15.40:0/4133806177 conn(0x55b8a9c42ff0 msgr2=0x55b8a9c48930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 -- 172.21.15.40:0/4133806177 shutdown_connections 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 -- 172.21.15.40:0/4133806177 wait complete. 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.313+0000 7f8926bc1b80 1 -- start start 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.314+0000 7f8926bc1b80 1 Processor -- start 2023-07-14T11:12:34.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.314+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.314+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c54a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9b3eac0 con 0x55b8a9c57d30 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9b3e7c0 con 0x55b8a9c4de00 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9a22920 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f8926bc1b80 10 client.0 _ll_register_callbacks cb 0x55b8a9b3c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f891179c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f891179c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39312/0 (socket says 172.21.15.40:39312) 2023-07-14T11:12:34.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.315+0000 7f891179c700 1 -- 172.21.15.40:0/422280514 learned_addr learned my addr 172.21.15.40:0/422280514 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 -- 172.21.15.40:0/422280514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 msgr2=0x55b8a9c57740 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f8910f9b700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 -- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 msgr2=0x55b8a9c54a30 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c54a30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 -- 172.21.15.40:0/422280514 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f890c1a0550 con 0x55b8a9c57d30 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f8910f9b700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:12:34.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f891179c700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f8904007970 tx=0x7f8904007d30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=88fb9d795a52c807 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.316+0000 7f88ebfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:34.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8904010760 con 0x55b8a9c57d30 2023-07-14T11:12:34.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f89040108c0 con 0x55b8a9c57d30 2023-07-14T11:12:34.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b8a9af84f0 con 0x55b8a9c57d30 2023-07-14T11:12:34.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8904010a80 con 0x55b8a9c57d30 2023-07-14T11:12:34.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f8926bc1b80 10 client.15831 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.317+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f890c10d0f0 con 0x55b8a9c57d30 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b8a9af84f0 con 0x55b8a9c57d30 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8904026420 con 0x55b8a9c57d30 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 10 client.15831 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 10 client.15831 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 20 client.15831 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9c676a0 0x55b8a9c69ab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 4 client.15831 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.318+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b8a9b97eb0 con 0x55b8a9c676a0 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.319+0000 7f8910f9b700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9c676a0 0x55b8a9c69ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.319+0000 7f8910f9b700 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9c676a0 0x55b8a9c69ab0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=94a40849e7aaa767 server_cookie=95ce06183aaeb1ca in_seq=0 out_seq=0 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.319+0000 7f88ebfff700 10 client.15831 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.320+0000 7f88ebfff700 1 -- 172.21.15.40:0/422280514 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55b8a9b97eb0 con 0x55b8a9c676a0 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.320+0000 7f88ebfff700 10 client.15831 handle_command_reply: tid=0 2023-07-14T11:12:34.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.320+0000 7f8926bc1b80 1 client.15831 shutdown 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.320+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 msgr2=0x55b8a9c550d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.320+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 crypto rx=0x7f8904007970 tx=0x7f8904007d30 comp rx=0 tx=0).stop 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 shutdown_connections 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9c676a0 0x55b8a9c69ab0 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c550d0 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/422280514 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c54a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 >> 172.21.15.40:0/422280514 conn(0x55b8a9c42ff0 msgr2=0x55b8a9c48520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f88ebfff700 0 client.15831 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:34.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 shutdown_connections 2023-07-14T11:12:34.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 1 -- 172.21.15.40:0/422280514 wait complete. 2023-07-14T11:12:34.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.321+0000 7f8926bc1b80 20 client.15831 trim_cache size 0 max 16384 2023-07-14T11:12:34.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.340+0000 7f8926bc1b80 1 Processor -- start 2023-07-14T11:12:34.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.340+0000 7f8926bc1b80 1 -- start start 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.340+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c541f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c54730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c4e9c0 con 0x55b8a9c57d30 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c4f8e0 con 0x55b8a9c4de00 2023-07-14T11:12:34.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c42ba0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f891179c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c54730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49328/0 (socket says 172.21.15.40:49328) 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 -- 172.21.15.40:0/3846650834 learned_addr learned my addr 172.21.15.40:0/3846650834 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 -- 172.21.15.40:0/3846650834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 msgr2=0x55b8a9c541f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c541f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 -- 172.21.15.40:0/3846650834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 msgr2=0x55b8a9c54730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c54730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.341+0000 7f8910f9b700 1 -- 172.21.15.40:0/3846650834 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b8a99b5c60 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.342+0000 7f8910f9b700 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f890000bc20 tx=0x7f8900095950 comp rx=0 tx=0).ready entity=mon.1 client_cookie=da46a5043ee07130 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.342+0000 7f88f77fe700 1 -- 172.21.15.40:0/3846650834 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8900097df0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.342+0000 7f88f77fe700 1 -- 172.21.15.40:0/3846650834 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f89000b1070 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.342+0000 7f88f77fe700 1 -- 172.21.15.40:0/3846650834 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8900099400 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- 172.21.15.40:0/3846650834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 msgr2=0x55b8a9c57370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 crypto rx=0x7f890000bc20 tx=0x7f8900095950 comp rx=0 tx=0).stop 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- 172.21.15.40:0/3846650834 shutdown_connections 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c57d30 0x55b8a9c54730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c57370 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/3846650834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c541f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- 172.21.15.40:0/3846650834 >> 172.21.15.40:0/3846650834 conn(0x55b8a9c42ff0 msgr2=0x55b8a9c4fd90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- 172.21.15.40:0/3846650834 shutdown_connections 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- 172.21.15.40:0/3846650834 wait complete. 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 -- start start 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.343+0000 7f8926bc1b80 1 Processor -- start 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c5c8f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5ce30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c5d370 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c5d4b0 con 0x55b8a9c4de00 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b8a9c5d670 con 0x55b8a9c57d30 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8926bc1b80 10 client.0 _ll_register_callbacks cb 0x7f8904009450 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8910f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39328/0 (socket says 172.21.15.40:39328) 2023-07-14T11:12:34.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f8910f9b700 1 -- 172.21.15.40:0/1418746696 learned_addr learned my addr 172.21.15.40:0/1418746696 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.344+0000 7f890bfff700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c5c8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 -- 172.21.15.40:0/1418746696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c57d30 msgr2=0x55b8a9c5ce30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5ce30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 -- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 msgr2=0x55b8a9c5c8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c5c8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 -- 172.21.15.40:0/1418746696 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f890c1942b0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8910f9b700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f89000ca3e0 tx=0x7f8900099920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c7bf2ae51038f293 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f88ebfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:34.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f89000993a0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f88f80054f0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8900099f10 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f890009a0d0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8926bc1b80 10 client.15837 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.345+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55b8a99f3250 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f88f80054f0 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8900016730 con 0x55b8a9c4d4f0 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 10 client.15837 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 10 client.15837 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 20 client.15837 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9cbc640 0x55b8a9cbea50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 4 client.15837 mds_command: new command op to 24521 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55b8a9b5a050 con 0x55b8a9cbc640 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.346+0000 7f890bfff700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9cbc640 0x55b8a9cbea50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.347+0000 7f890bfff700 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9cbc640 0x55b8a9cbea50 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=3c39690b495a00e6 server_cookie=b6f37923f9613942 in_seq=0 out_seq=0 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.347+0000 7f88ebfff700 10 client.15837 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.347+0000 7f88ebfff700 1 -- 172.21.15.40:0/1418746696 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55b8a9b5a050 con 0x55b8a9cbc640 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.347+0000 7f88ebfff700 10 client.15837 handle_command_reply: tid=0 2023-07-14T11:12:34.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.347+0000 7f8926bc1b80 1 client.15837 shutdown 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 msgr2=0x55b8a9c5e670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 crypto rx=0x7f89000ca3e0 tx=0x7f8900099920 comp rx=0 tx=0).stop 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 shutdown_connections 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55b8a9cbc640 0x55b8a9cbea50 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b8a9c57d30 0x55b8a9c5ce30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b8a9c4d4f0 0x55b8a9c5e670 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 --2- 172.21.15.40:0/1418746696 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b8a9c4de00 0x55b8a9c5c8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 >> 172.21.15.40:0/1418746696 conn(0x55b8a9c42ff0 msgr2=0x55b8a9c4c440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f88ebfff700 0 client.15837 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 shutdown_connections 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 1 -- 172.21.15.40:0/1418746696 wait complete. 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.348+0000 7f8926bc1b80 20 client.15837 trim_cache size 0 max 16384 2023-07-14T11:12:34.357 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:34.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 msgr2=0x7f88fc0b4930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 0x7f88fc0b4930 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f88f800b650 tx=0x7f88f801b000 comp rx=0 tx=0).stop 2023-07-14T11:12:34.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 msgr2=0x7f890c199560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f890000bc50 tx=0x7f8900008510 comp rx=0 tx=0).stop 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 shutdown_connections 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88fc0b2480 0x7f88fc0b4930 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f890c109fc0 0x7f890c193640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f890c100aa0 0x7f890c199560 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 --2- 172.21.15.40:0/1166492768 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f890c1013b0 0x7f890c199aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 >> 172.21.15.40:0/1166492768 conn(0x7f890c0745d0 msgr2=0x7f890c0fd380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 shutdown_connections 2023-07-14T11:12:34.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.349+0000 7f8911f9d700 1 -- 172.21.15.40:0/1166492768 wait complete. 2023-07-14T11:12:34.369 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2023-07-14T11:12:34.445 INFO:teuthology.orchestra.run.smithi040.stderr:3600000 of 50000000 tuples (7%) done (elapsed 8.94 s, remaining 115.20 s) 2023-07-14T11:12:34.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 -- 172.21.15.40:0/1634283468 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc40731c0 msgr2=0x7f2cc410bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/1634283468 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc40731c0 0x7f2cc410bde0 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 crypto rx=0x7f2cb8009990 tx=0x7f2cb8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 -- 172.21.15.40:0/1634283468 shutdown_connections 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/1634283468 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc410c320 0x7f2cc410e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/1634283468 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40728b0 0x7f2cc4072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/1634283468 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc40731c0 0x7f2cc410bde0 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 -- 172.21.15.40:0/1634283468 >> 172.21.15.40:0/1634283468 conn(0x7f2cc40fb5e0 msgr2=0x7f2cc40fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 -- 172.21.15.40:0/1634283468 shutdown_connections 2023-07-14T11:12:34.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.602+0000 7f2cc99ff700 1 -- 172.21.15.40:0/1634283468 wait complete. 2023-07-14T11:12:34.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.604+0000 7f2cc99ff700 1 Processor -- start 2023-07-14T11:12:34.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.604+0000 7f2cc99ff700 1 -- start start 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc40728b0 0x7f2cc4197dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 0x7f2cc4198310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 0x7f2cc419c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2cc419cdb0 con 0x7f2cc410c320 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2cc419cf20 con 0x7f2cc40731c0 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc99ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2cc419d0e0 con 0x7f2cc40728b0 2023-07-14T11:12:34.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 0x7f2cc419c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 0x7f2cc419c770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39350/0 (socket says 172.21.15.40:39350) 2023-07-14T11:12:34.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc37fe700 1 -- 172.21.15.40:0/4047889921 learned_addr learned my addr 172.21.15.40:0/4047889921 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc40728b0 0x7f2cc4197dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.605+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 0x7f2cc4198310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 -- 172.21.15.40:0/4047889921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc40728b0 msgr2=0x7f2cc4197dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc40728b0 0x7f2cc4197dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 -- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 msgr2=0x7f2cc419c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 0x7f2cc419c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 -- 172.21.15.40:0/4047889921 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2cb4009520 con 0x7f2cc40731c0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 0x7f2cc4198310 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f2cb8000c00 tx=0x7f2cb8005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc89fd700 1 -- 172.21.15.40:0/4047889921 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb801d070 con 0x7f2cc40731c0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc89fd700 1 -- 172.21.15.40:0/4047889921 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2cb8021410 con 0x7f2cc40731c0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc89fd700 1 -- 172.21.15.40:0/4047889921 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb8017650 con 0x7f2cc40731c0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2cb80095f0 con 0x7f2cc40731c0 2023-07-14T11:12:34.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.606+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2cc41a4a20 con 0x7f2cc40731c0 2023-07-14T11:12:34.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.609+0000 7f2cc89fd700 1 -- 172.21.15.40:0/4047889921 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f2cb80217f0 con 0x7f2cc40731c0 2023-07-14T11:12:34.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cc89fd700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 0x7f2cac0a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cdd3eeb80 1 Processor -- start 2023-07-14T11:12:34.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cc89fd700 1 -- 172.21.15.40:0/4047889921 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2cb80ce450 con 0x7f2cc40731c0 2023-07-14T11:12:34.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cdd3eeb80 1 -- start start 2023-07-14T11:12:34.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 0x7f2cac0a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.611+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ade66a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf3eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93aceccb0 con 0x55c93ade6be0 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ad3fa40 con 0x55c93ade62d0 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ad38ed0 con 0x55c93adf1ad0 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc2ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ade66a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39358/0 (socket says 172.21.15.40:39358) 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 -- 172.21.15.40:0/3811075441 learned_addr learned my addr 172.21.15.40:0/3811075441 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc37fe700 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf3eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 -- 172.21.15.40:0/3811075441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 msgr2=0x55c93adf3eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf3eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 -- 172.21.15.40:0/3811075441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 msgr2=0x55c93ade66a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ade66a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.612+0000 7f2cc27fc700 1 -- 172.21.15.40:0/3811075441 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c93ab66de0 con 0x55c93ade6be0 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.613+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f2cb800b2e0 tx=0x7f2cb8097530 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6405032a7a1358a0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.613+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 0x7f2cac0a3880 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f2cb4009fd0 tx=0x7f2cb4009ce0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.613+0000 7f2ca9ffb700 1 -- 172.21.15.40:0/3811075441 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb809aa30 con 0x55c93ade6be0 2023-07-14T11:12:34.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.613+0000 7f2ca9ffb700 1 -- 172.21.15.40:0/3811075441 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2cb8021e90 con 0x55c93ade6be0 2023-07-14T11:12:34.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.613+0000 7f2ca9ffb700 1 -- 172.21.15.40:0/3811075441 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb809b590 con 0x55c93ade6be0 2023-07-14T11:12:34.619 INFO:teuthology.orchestra.run.smithi040.stderr:3700000 of 50000000 tuples (7%) done (elapsed 9.11 s, remaining 113.98 s) 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/3811075441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 msgr2=0x55c93adf1590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 crypto rx=0x7f2cb800b2e0 tx=0x7f2cb8097530 comp rx=0 tx=0).stop 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/3811075441 shutdown_connections 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf3eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ade66a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/3811075441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf1590 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/3811075441 >> 172.21.15.40:0/3811075441 conn(0x55c93addcda0 msgr2=0x55c93ade26a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/3811075441 shutdown_connections 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.614+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/3811075441 wait complete. 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.615+0000 7f2cdd3eeb80 1 -- start start 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.615+0000 7f2cdd3eeb80 1 Processor -- start 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93adf6bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf0a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93acd8ac0 con 0x55c93ade6be0 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93acd87c0 con 0x55c93ade62d0 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93abbc920 con 0x55c93adf1ad0 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cdd3eeb80 10 client.0 _ll_register_callbacks cb 0x55c93acd6760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cc27fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cc27fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39370/0 (socket says 172.21.15.40:39370) 2023-07-14T11:12:34.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cc27fc700 1 -- 172.21.15.40:0/2661950679 learned_addr learned my addr 172.21.15.40:0/2661950679 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.616+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93adf6bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc37fe700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf0a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 msgr2=0x55c93adf0a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf0a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/2661950679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 msgr2=0x55c93adf7100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/2661950679 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2cc41a4a20 con 0x55c93ade62d0 2023-07-14T11:12:34.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93adf6bc0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f2cb400f6a0 tx=0x7f2cb401d000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=51de42c16669ab02 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2ca2ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.617+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb4013170 con 0x55c93ade62d0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c93ac924f0 con 0x55c93ade62d0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2cb4025030 con 0x55c93ade62d0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2cb40132d0 con 0x55c93ade62d0 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 10 client.25844 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55c93ab4fc60 con 0x55c93ade62d0 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c93ac924f0 con 0x55c93ade62d0 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2cb4010550 con 0x55c93ade62d0 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 10 client.25844 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 10 client.25844 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 20 client.25844 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae01040 0x55c93ae03450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 4 client.25844 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:34.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.618+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c93ad31eb0 con 0x55c93ae01040 2023-07-14T11:12:34.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.619+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae01040 0x55c93ae03450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.619+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae01040 0x55c93ae03450 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=bd93410526d96bbd server_cookie=ba446d91dbdc2e75 in_seq=0 out_seq=0 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.619+0000 7f2ca2ffd700 10 client.25844 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/2661950679 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c93ad31eb0 con 0x55c93ae01040 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2ca2ffd700 10 client.25844 handle_command_reply: tid=0 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 client.25844 shutdown 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 msgr2=0x55c93adf6bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93adf6bc0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f2cb400f6a0 tx=0x7f2cb401d000 comp rx=0 tx=0).stop 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 shutdown_connections 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae01040 0x55c93ae03450 crc :-1 s=READY pgs=34 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93adf1ad0 0x55c93adf0a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2ca2ffd700 0 client.25844 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93adf6bc0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2661950679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93ade6be0 0x55c93adf7100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 >> 172.21.15.40:0/2661950679 conn(0x55c93addcda0 msgr2=0x55c93ade2230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 shutdown_connections 2023-07-14T11:12:34.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.620+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2661950679 wait complete. 2023-07-14T11:12:34.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.621+0000 7f2cdd3eeb80 20 client.25844 trim_cache size 0 max 16384 2023-07-14T11:12:34.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 Processor -- start 2023-07-14T11:12:34.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 -- start start 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae03420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ade8c70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93addca00 con 0x55c93adf1ad0 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ade81a0 con 0x55c93ade62d0 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.635+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93adec650 con 0x55c93ade6be0 2023-07-14T11:12:34.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39376/0 (socket says 172.21.15.40:39376) 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc37fe700 1 -- 172.21.15.40:0/2017454675 learned_addr learned my addr 172.21.15.40:0/2017454675 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ade8c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae03420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 -- 172.21.15.40:0/2017454675 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 msgr2=0x55c93ae03420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae03420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 -- 172.21.15.40:0/2017454675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 msgr2=0x55c93adf6750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 -- 172.21.15.40:0/2017454675 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c93ab8d250 con 0x55c93ade6be0 2023-07-14T11:12:34.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc37fe700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.636+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ade8c70 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f2cb809add0 tx=0x7f2cb8005840 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ba28438eea38807a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2caa7fc700 1 -- 172.21.15.40:0/2017454675 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb80cb6b0 con 0x55c93ade6be0 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2caa7fc700 1 -- 172.21.15.40:0/2017454675 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2cb80a3490 con 0x55c93ade6be0 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2caa7fc700 1 -- 172.21.15.40:0/2017454675 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb80ce170 con 0x55c93ade6be0 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2017454675 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 msgr2=0x55c93ade8c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ade8c70 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7f2cb809add0 tx=0x7f2cb8005840 comp rx=0 tx=0).stop 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2017454675 shutdown_connections 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93adf6750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae03420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/2017454675 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ade8c70 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.637+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2017454675 >> 172.21.15.40:0/2017454675 conn(0x55c93addcda0 msgr2=0x55c93ae07db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.638+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2017454675 shutdown_connections 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.638+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/2017454675 wait complete. 2023-07-14T11:12:34.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.638+0000 7f2cdd3eeb80 1 -- start start 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.638+0000 7f2cdd3eeb80 1 Processor -- start 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ae495c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93ae4dee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ae4e420 con 0x55c93adf1ad0 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ae4e560 con 0x55c93ade62d0 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c93ae4e720 con 0x55c93ade6be0 2023-07-14T11:12:34.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cdd3eeb80 10 client.0 _ll_register_callbacks cb 0x7f2cb40134c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60540/0 (socket says 172.21.15.40:60540) 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/871003267 learned_addr learned my addr 172.21.15.40:0/871003267 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc37fe700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93ae4dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ae495c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 msgr2=0x55c93ae495c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ae495c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/871003267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 msgr2=0x55c93ae4dee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93ae4dee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.639+0000 7f2cc2ffd700 1 -- 172.21.15.40:0/871003267 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2cbc0145b0 con 0x55c93ade62d0 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2cc2ffd700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f2cb400f6a0 tx=0x7f2cb4010ac0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b9438b2c9dc10d3c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2ca2ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2cb400fb70 con 0x55c93ade62d0 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f2cb4025040 con 0x55c93ade62d0 2023-07-14T11:12:34.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2cb8021e90 con 0x55c93ade62d0 2023-07-14T11:12:34.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2cb4010b30 con 0x55c93ade62d0 2023-07-14T11:12:34.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2cdd3eeb80 10 client.25850 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.640+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f2cb80b2040 con 0x55c93ade62d0 2023-07-14T11:12:34.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2cb8021e90 con 0x55c93ade62d0 2023-07-14T11:12:34.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2cb4003b40 con 0x55c93ade62d0 2023-07-14T11:12:34.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 10 client.25850 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 10 client.25850 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 20 client.25850 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae55df0 0x55c93ae581d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 4 client.25850 mds_command: new command op to 24523 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c93acf4050 con 0x55c93ae55df0 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.641+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae55df0 0x55c93ae581d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2ca2ffd700 10 client.25850 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:34.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cc27fc700 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae55df0 0x55c93ae581d0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=cb34bfacbb580e4a server_cookie=c09a47ad9f49d6b4 in_seq=0 out_seq=0 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2ca2ffd700 1 -- 172.21.15.40:0/871003267 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c93acf4050 con 0x55c93ae55df0 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2ca2ffd700 10 client.25850 handle_command_reply: tid=0 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 client.25850 shutdown 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 msgr2=0x55c93ae49080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f2cb400f6a0 tx=0x7f2cb4010ac0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 shutdown_connections 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c93ae55df0 0x55c93ae581d0 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c93adf1ad0 0x55c93ae4dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c93ade62d0 0x55c93ae49080 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 --2- 172.21.15.40:0/871003267 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c93ade6be0 0x55c93ae495c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2ca2ffd700 0 client.25850 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 >> 172.21.15.40:0/871003267 conn(0x55c93addcda0 msgr2=0x55c93ade8c70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 shutdown_connections 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.642+0000 7f2cdd3eeb80 1 -- 172.21.15.40:0/871003267 wait complete. 2023-07-14T11:12:34.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cdd3eeb80 20 client.25850 trim_cache size 0 max 16384 2023-07-14T11:12:34.653 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:34.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 msgr2=0x7f2cac0a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 0x7f2cac0a3880 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f2cb4009fd0 tx=0x7f2cb4009ce0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 msgr2=0x7f2cc4198310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 0x7f2cc4198310 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f2cb8000c00 tx=0x7f2cb8005740 comp rx=0 tx=0).stop 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 shutdown_connections 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f2cac0a13d0 0x7f2cac0a3880 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2cc410c320 0x7f2cc419c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f2cc40728b0 0x7f2cc4197dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 --2- 172.21.15.40:0/4047889921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2cc40731c0 0x7f2cc4198310 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 >> 172.21.15.40:0/4047889921 conn(0x7f2cc40fb5e0 msgr2=0x7f2cc410e3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.643+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 shutdown_connections 2023-07-14T11:12:34.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.644+0000 7f2cc99ff700 1 -- 172.21.15.40:0/4047889921 wait complete. 2023-07-14T11:12:34.662 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:34 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/405072273' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:34.664 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2023-07-14T11:12:34.886 INFO:teuthology.orchestra.run.smithi040.stderr:3800000 of 50000000 tuples (7%) done (elapsed 9.38 s, remaining 114.02 s) 2023-07-14T11:12:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 -- 172.21.15.40:0/1386880123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad80728b0 msgr2=0x7f5ad8072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1386880123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad80728b0 0x7f5ad8072c80 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7f5ac8009990 tx=0x7f5ac8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 -- 172.21.15.40:0/1386880123 shutdown_connections 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1386880123 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad810c320 0x7f5ad810e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1386880123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad80728b0 0x7f5ad8072c80 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1386880123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad810bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 -- 172.21.15.40:0/1386880123 >> 172.21.15.40:0/1386880123 conn(0x7f5ad80fb5e0 msgr2=0x7f5ad80fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 -- 172.21.15.40:0/1386880123 shutdown_connections 2023-07-14T11:12:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.898+0000 7f5add9ff700 1 -- 172.21.15.40:0/1386880123 wait complete. 2023-07-14T11:12:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.899+0000 7f5add9ff700 1 Processor -- start 2023-07-14T11:12:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 -- start start 2023-07-14T11:12:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 0x7f5ad8197e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad8198350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 0x7f5ad819c7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ad819cdf0 con 0x7f5ad80731c0 2023-07-14T11:12:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ad819cf60 con 0x7f5ad80728b0 2023-07-14T11:12:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.900+0000 7f5add9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5ad819d120 con 0x7f5ad810c320 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad67fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad8198350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 0x7f5ad8197e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad67fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad8198350 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39410/0 (socket says 172.21.15.40:39410) 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad67fc700 1 -- 172.21.15.40:0/1447600848 learned_addr learned my addr 172.21.15.40:0/1447600848 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 0x7f5ad819c7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/1447600848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 msgr2=0x7f5ad819c7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 0x7f5ad819c7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 msgr2=0x7f5ad8198350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad8198350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/1447600848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5ad0009c40 con 0x7f5ad80728b0 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 0x7f5ad819c7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.901+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 0x7f5ad8197e10 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5ac8000c00 tx=0x7f5ac8005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.902+0000 7f5adc9fd700 1 -- 172.21.15.40:0/1447600848 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ac801d070 con 0x7f5ad80728b0 2023-07-14T11:12:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.902+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5ac80095f0 con 0x7f5ad80728b0 2023-07-14T11:12:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.902+0000 7f5adc9fd700 1 -- 172.21.15.40:0/1447600848 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ac8021410 con 0x7f5ad80728b0 2023-07-14T11:12:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.902+0000 7f5adc9fd700 1 -- 172.21.15.40:0/1447600848 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ac8017650 con 0x7f5ad80728b0 2023-07-14T11:12:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.902+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ad81a4a60 con 0x7f5ad80728b0 2023-07-14T11:12:34.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.907+0000 7f5adc9fd700 1 -- 172.21.15.40:0/1447600848 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f5ac80217f0 con 0x7f5ad80728b0 2023-07-14T11:12:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 Processor -- start 2023-07-14T11:12:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 -- start start 2023-07-14T11:12:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c276a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c32590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5adc9fd700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 0x7f5ac00a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c34eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4b2dcb0 con 0x5618a4c27be0 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4b80a40 con 0x5618a4c32ad0 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.908+0000 7f5af124db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4b79ed0 con 0x5618a4c272d0 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5adc9fd700 1 -- 172.21.15.40:0/1447600848 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5ac80ce5c0 con 0x7f5ad80728b0 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c32590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c32590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39418/0 (socket says 172.21.15.40:39418) 2023-07-14T11:12:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/170017153 learned_addr learned my addr 172.21.15.40:0/170017153 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c276a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c34eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 0x7f5ac00a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 -- 172.21.15.40:0/170017153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 msgr2=0x5618a4c276a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c276a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 -- 172.21.15.40:0/170017153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 msgr2=0x5618a4c32590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c32590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.909+0000 7f5ad77fe700 1 -- 172.21.15.40:0/170017153 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5618a49a7de0 con 0x5618a4c32ad0 2023-07-14T11:12:34.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c34eb0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f5ad0009fd0 tx=0x7f5ad000b040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c6e4b66a07785525 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 0x7f5ac00a3800 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f5ad8199420 tx=0x7f5acc009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5abdffb700 1 -- 172.21.15.40:0/170017153 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad000e040 con 0x5618a4c32ad0 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5abdffb700 1 -- 172.21.15.40:0/170017153 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ad0019e90 con 0x5618a4c32ad0 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5abdffb700 1 -- 172.21.15.40:0/170017153 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad0004020 con 0x5618a4c32ad0 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5af124db80 1 -- 172.21.15.40:0/170017153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 msgr2=0x5618a4c34eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.910+0000 7f5af124db80 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c34eb0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f5ad0009fd0 tx=0x7f5ad000b040 comp rx=0 tx=0).stop 2023-07-14T11:12:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 -- 172.21.15.40:0/170017153 shutdown_connections 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c34eb0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c276a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 --2- 172.21.15.40:0/170017153 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c32590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 -- 172.21.15.40:0/170017153 >> 172.21.15.40:0/170017153 conn(0x5618a4c1dda0 msgr2=0x5618a4c236a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 -- 172.21.15.40:0/170017153 shutdown_connections 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 -- 172.21.15.40:0/170017153 wait complete. 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.911+0000 7f5af124db80 1 -- start start 2023-07-14T11:12:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 Processor -- start 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2f820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c2ff60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4b19ac0 con 0x5618a4c27be0 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4b197c0 con 0x5618a4c32ad0 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a49fd920 con 0x5618a4c272d0 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5af124db80 10 client.0 _ll_register_callbacks cb 0x5618a4b17760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5ad77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c2ff60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49438/0 (socket says 172.21.15.40:49438) 2023-07-14T11:12:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/295526669 learned_addr learned my addr 172.21.15.40:0/295526669 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.912+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2f820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/295526669 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 msgr2=0x5618a4c2ff60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c2ff60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 msgr2=0x5618a4c2f820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2f820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/295526669 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5ad0012c50 con 0x5618a4c272d0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f5ac809af40 tx=0x7f5ac800fdb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dd44509beb02948c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ab6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ac809f270 con 0x5618a4c272d0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5618a4ad34f0 con 0x5618a4c272d0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5ac80a3600 con 0x5618a4c272d0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5ac80ce200 con 0x5618a4c272d0 2023-07-14T11:12:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5af124db80 10 client.25849 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.913+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5618a4990c60 con 0x5618a4c272d0 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ac80b9020 con 0x5618a4c272d0 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f5ac809b4a0 con 0x5618a4c272d0 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 10 client.25849 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 10 client.25849 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 20 client.25849 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c41f40 0x5618a4c44350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 4 client.25849 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5618a4b72eb0 con 0x5618a4c41f40 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.914+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c41f40 0x5618a4c44350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.915+0000 7f5ab6ffd700 10 client.25849 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.915+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c41f40 0x5618a4c44350 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=5600288070c6d575 server_cookie=f000af0bb9c94424 in_seq=0 out_seq=0 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.915+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/295526669 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5618a4b72eb0 con 0x5618a4c41f40 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.915+0000 7f5ab6ffd700 10 client.25849 handle_command_reply: tid=0 2023-07-14T11:12:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.915+0000 7f5af124db80 1 client.25849 shutdown 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 msgr2=0x5618a4c32530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 crypto rx=0x7f5ac809af40 tx=0x7f5ac800fdb0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 shutdown_connections 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c41f40 0x5618a4c44350 crc :-1 s=READY pgs=35 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c32ad0 0x5618a4c2ff60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c32530 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5ab6ffd700 0 client.25849 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 --2- 172.21.15.40:0/295526669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2f820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 >> 172.21.15.40:0/295526669 conn(0x5618a4c1dda0 msgr2=0x5618a4c23290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 shutdown_connections 2023-07-14T11:12:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 1 -- 172.21.15.40:0/295526669 wait complete. 2023-07-14T11:12:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.916+0000 7f5af124db80 20 client.25849 trim_cache size 0 max 16384 2023-07-14T11:12:34.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 Processor -- start 2023-07-14T11:12:34.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 -- start start 2023-07-14T11:12:34.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c272d0 0x5618a4c36ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2efe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c24500 con 0x5618a4c27be0 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c28dd0 con 0x5618a4c272d0 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5af124db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c2dd80 con 0x5618a4c32ad0 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5ad77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5ad67fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2efe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5ad77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49446/0 (socket says 172.21.15.40:49446) 2023-07-14T11:12:34.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5ad77fe700 1 -- 172.21.15.40:0/2922541345 learned_addr learned my addr 172.21.15.40:0/2922541345 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.930+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c272d0 0x5618a4c36ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 -- 172.21.15.40:0/2922541345 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c272d0 msgr2=0x5618a4c36ed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c272d0 0x5618a4c36ed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 -- 172.21.15.40:0/2922541345 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 msgr2=0x5618a4c2efe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2efe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 -- 172.21.15.40:0/2922541345 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5618a4c1f5e0 con 0x5618a4c32ad0 2023-07-14T11:12:34.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f5ad001c040 tx=0x7f5ad0009110 comp rx=0 tx=0).ready entity=mon.1 client_cookie=87d2aa3e9005886e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5abe7fc700 1 -- 172.21.15.40:0/2922541345 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad0019c90 con 0x5618a4c32ad0 2023-07-14T11:12:34.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5abe7fc700 1 -- 172.21.15.40:0/2922541345 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5ad00122a0 con 0x5618a4c32ad0 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.931+0000 7f5abe7fc700 1 -- 172.21.15.40:0/2922541345 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5ad0004020 con 0x5618a4c32ad0 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 -- 172.21.15.40:0/2922541345 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 msgr2=0x5618a4c2f520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 crypto rx=0x7f5ad001c040 tx=0x7f5ad0009110 comp rx=0 tx=0).stop 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 -- 172.21.15.40:0/2922541345 shutdown_connections 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c32ad0 0x5618a4c2f520 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c272d0 0x5618a4c36ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 --2- 172.21.15.40:0/2922541345 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c27be0 0x5618a4c2efe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.932+0000 7f5af124db80 1 -- 172.21.15.40:0/2922541345 >> 172.21.15.40:0/2922541345 conn(0x5618a4c1dda0 msgr2=0x5618a4c231b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.933+0000 7f5af124db80 1 -- 172.21.15.40:0/2922541345 shutdown_connections 2023-07-14T11:12:34.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.933+0000 7f5af124db80 1 -- 172.21.15.40:0/2922541345 wait complete. 2023-07-14T11:12:34.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.933+0000 7f5af124db80 1 -- start start 2023-07-14T11:12:34.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 Processor -- start 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c36d80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 0x5618a4c35040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c32ad0 0x5618a4c355c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c35cd0 con 0x5618a4c32ad0 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c35e40 con 0x5618a4c27be0 2023-07-14T11:12:34.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5618a4c95dd0 con 0x5618a4c272d0 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.934+0000 7f5af124db80 10 client.0 _ll_register_callbacks cb 0x5618a4b17760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c36d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c36d80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49460/0 (socket says 172.21.15.40:49460) 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad6ffd700 1 -- 172.21.15.40:0/179097626 learned_addr learned my addr 172.21.15.40:0/179097626 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 0x5618a4c35040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad77fe700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c32ad0 0x5618a4c355c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 -- 172.21.15.40:0/179097626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 msgr2=0x5618a4c36d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c36d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 -- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c32ad0 msgr2=0x5618a4c355c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c32ad0 0x5618a4c355c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 -- 172.21.15.40:0/179097626 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5618a49a7de0 con 0x5618a4c27be0 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ad67fc700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 0x5618a4c35040 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x5618a4c44bf0 tx=0x7f5acc013040 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8ec58ee8dd3ee858 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ab6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.935+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5acc01e2d0 con 0x5618a4c27be0 2023-07-14T11:12:34.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5618a4b35050 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5acc025130 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5acc01e430 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 10 client.25868 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5618a49ce250 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5618a4b35050 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f5acc02e470 con 0x5618a4c27be0 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 10 client.25868 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:34.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 10 client.25868 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 20 client.25868 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c97480 0x5618a4c99860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 4 client.25868 mds_command: new command op to 24565 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.936+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x5618a49c8f70 con 0x5618a4c97480 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.937+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c97480 0x5618a4c99860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.937+0000 7f5ab6ffd700 10 client.25868 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.937+0000 7f5ad6ffd700 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c97480 0x5618a4c99860 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=c420adf5ea7a4321 server_cookie=a0b3b9c46c05b0af in_seq=0 out_seq=0 2023-07-14T11:12:34.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.937+0000 7f5ab6ffd700 1 -- 172.21.15.40:0/179097626 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x5618a49c8f70 con 0x5618a4c97480 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.937+0000 7f5ab6ffd700 10 client.25868 handle_command_reply: tid=0 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 client.25868 shutdown 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 msgr2=0x5618a4c35040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 0x5618a4c35040 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x5618a4c44bf0 tx=0x7f5acc013040 comp rx=0 tx=0).stop 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 shutdown_connections 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x5618a4c97480 0x5618a4c99860 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5618a4c32ad0 0x5618a4c355c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5618a4c272d0 0x5618a4c36d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5ab6ffd700 0 client.25868 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 --2- 172.21.15.40:0/179097626 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5618a4c27be0 0x5618a4c35040 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 >> 172.21.15.40:0/179097626 conn(0x5618a4c1dda0 msgr2=0x5618a4c3d680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 shutdown_connections 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 1 -- 172.21.15.40:0/179097626 wait complete. 2023-07-14T11:12:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.938+0000 7f5af124db80 20 client.25868 trim_cache size 0 max 16384 2023-07-14T11:12:34.948 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 msgr2=0x7f5ac00a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 0x7f5ac00a3800 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f5ad8199420 tx=0x7f5acc009260 comp rx=0 tx=0).stop 2023-07-14T11:12:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 msgr2=0x7f5ad8197e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 0x7f5ad8197e10 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7f5ac8000c00 tx=0x7f5ac8005740 comp rx=0 tx=0).stop 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 shutdown_connections 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5ac00a1350 0x7f5ac00a3800 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5ad810c320 0x7f5ad819c7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5ad80728b0 0x7f5ad8197e10 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 --2- 172.21.15.40:0/1447600848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5ad80731c0 0x7f5ad8198350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 >> 172.21.15.40:0/1447600848 conn(0x7f5ad80fb5e0 msgr2=0x7f5ad810e410 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 shutdown_connections 2023-07-14T11:12:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:34.939+0000 7f5add9ff700 1 -- 172.21.15.40:0/1447600848 wait complete. 2023-07-14T11:12:34.961 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2023-07-14T11:12:35.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:34 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/405072273' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:35.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:34 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/405072273' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:12:35.194 INFO:teuthology.orchestra.run.smithi040.stderr:3900000 of 50000000 tuples (7%) done (elapsed 9.69 s, remaining 114.50 s) 2023-07-14T11:12:35.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 -- 172.21.15.40:0/621518385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a00731c0 msgr2=0x7f80a010bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 --2- 172.21.15.40:0/621518385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a00731c0 0x7f80a010bdd0 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 crypto rx=0x7f80940098f0 tx=0x7f8094009c00 comp rx=0 tx=0).stop 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 -- 172.21.15.40:0/621518385 shutdown_connections 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 --2- 172.21.15.40:0/621518385 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a010c310 0x7f80a010e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 --2- 172.21.15.40:0/621518385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.200+0000 7f80a5bff700 1 --2- 172.21.15.40:0/621518385 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a00731c0 0x7f80a010bdd0 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.201+0000 7f80a5bff700 1 -- 172.21.15.40:0/621518385 >> 172.21.15.40:0/621518385 conn(0x7f80a00fb5f0 msgr2=0x7f80a00fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.201+0000 7f80a5bff700 1 -- 172.21.15.40:0/621518385 shutdown_connections 2023-07-14T11:12:35.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.201+0000 7f80a5bff700 1 -- 172.21.15.40:0/621518385 wait complete. 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 Processor -- start 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 -- start start 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a00731c0 0x7f80a0198310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a010c310 0x7f80a019c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80a019cdb0 con 0x7f80a00728b0 2023-07-14T11:12:35.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80a019cf20 con 0x7f80a00731c0 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.202+0000 7f80a5bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80a019d0e0 con 0x7f80a010c310 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809effd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a00731c0 0x7f80a0198310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39462/0 (socket says 172.21.15.40:39462) 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 -- 172.21.15.40:0/2082172661 learned_addr learned my addr 172.21.15.40:0/2082172661 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809ffff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a010c310 0x7f80a019c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 -- 172.21.15.40:0/2082172661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a010c310 msgr2=0x7f80a019c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a010c310 0x7f80a019c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 -- 172.21.15.40:0/2082172661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a00731c0 msgr2=0x7f80a0198310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a00731c0 0x7f80a0198310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 -- 172.21.15.40:0/2082172661 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8090009520 con 0x7f80a00728b0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.203+0000 7f809f7fe700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f809000eb30 tx=0x7f809000ee40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.204+0000 7f809cff9700 1 -- 172.21.15.40:0/2082172661 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8090009880 con 0x7f80a00728b0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.204+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8094009520 con 0x7f80a00728b0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.204+0000 7f809cff9700 1 -- 172.21.15.40:0/2082172661 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f80900044e0 con 0x7f80a00728b0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.204+0000 7f809cff9700 1 -- 172.21.15.40:0/2082172661 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8090003d80 con 0x7f80a00728b0 2023-07-14T11:12:35.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.204+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80a01a4a20 con 0x7f80a00728b0 2023-07-14T11:12:35.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 Processor -- start 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 -- start start 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870abe0 0x55c028715590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f809cff9700 1 -- 172.21.15.40:0/2082172661 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7f8090003f20 con 0x7f80a00728b0 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028717eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c028610cb0 con 0x55c02870a2d0 2023-07-14T11:12:35.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c028663a40 con 0x55c02870abe0 2023-07-14T11:12:35.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c02865ced0 con 0x55c028715ad0 2023-07-14T11:12:35.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.209+0000 7f809ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39478/0 (socket says 172.21.15.40:39478) 2023-07-14T11:12:35.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 -- 172.21.15.40:0/3545157974 learned_addr learned my addr 172.21.15.40:0/3545157974 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:35.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809effd700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028717eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809f7fe700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870abe0 0x55c028715590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 -- 172.21.15.40:0/3545157974 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 msgr2=0x55c028717eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028717eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 -- 172.21.15.40:0/3545157974 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870abe0 msgr2=0x55c028715590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870abe0 0x55c028715590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 -- 172.21.15.40:0/3545157974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c02848ade0 con 0x55c02870a2d0 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809effd700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028717eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.210+0000 7f809ffff700 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f8098011670 tx=0x7f80980099e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3a170c9096185227 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f8085ffb700 1 -- 172.21.15.40:0/3545157974 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f809800e040 con 0x55c02870a2d0 2023-07-14T11:12:35.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f8085ffb700 1 -- 172.21.15.40:0/3545157974 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f809800f040 con 0x55c02870a2d0 2023-07-14T11:12:35.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f8085ffb700 1 -- 172.21.15.40:0/3545157974 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8098004610 con 0x55c02870a2d0 2023-07-14T11:12:35.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f809cff9700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 0x7f80880a3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f809cff9700 1 -- 172.21.15.40:0/2082172661 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8090014070 con 0x7f80a00728b0 2023-07-14T11:12:35.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.211+0000 7f809f7fe700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 0x7f80880a3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 -- 172.21.15.40:0/3545157974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 msgr2=0x55c02870a6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 crypto rx=0x7f8098011670 tx=0x7f80980099e0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 -- 172.21.15.40:0/3545157974 shutdown_connections 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028717eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c02870a6a0 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/3545157974 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870abe0 0x55c028715590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 -- 172.21.15.40:0/3545157974 >> 172.21.15.40:0/3545157974 conn(0x55c028700da0 msgr2=0x55c0287066a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 -- 172.21.15.40:0/3545157974 shutdown_connections 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f80b95d7b80 1 -- 172.21.15.40:0/3545157974 wait complete. 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.212+0000 7f809f7fe700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 0x7f80880a3880 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f80900caa80 tx=0x7f80900d0f00 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 -- start start 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 Processor -- start 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870abe0 0x55c028712850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028712f90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0285fcac0 con 0x55c02870abe0 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0285fc7c0 con 0x55c02870a2d0 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.213+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0284e0920 con 0x55c028715ad0 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f80b95d7b80 10 client.0 _ll_register_callbacks cb 0x55c0285fa760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60610/0 (socket says 172.21.15.40:60610) 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 -- 172.21.15.40:0/4052997376 learned_addr learned my addr 172.21.15.40:0/4052997376 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:35.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809effd700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870abe0 0x55c028712850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028712f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 -- 172.21.15.40:0/4052997376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 msgr2=0x55c028712f90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028712f90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 -- 172.21.15.40:0/4052997376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870abe0 msgr2=0x55c028712850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870abe0 0x55c028712850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.214+0000 7f809ffff700 1 -- 172.21.15.40:0/4052997376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80a010f390 con 0x55c02870a2d0 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f809ffff700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f8098009bc0 tx=0x7f8098011f90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c4d6dfc45aa1550d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f807effd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f809800e070 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8098004ad0 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c0285b64f0 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8098006160 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f80b95d7b80 10 client.25880 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x7f8098011350 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.215+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c0285b64f0 con 0x55c02870a2d0 2023-07-14T11:12:35.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f80980079c0 con 0x55c02870a2d0 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 10 client.25880 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 10 client.25880 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 20 client.25880 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287253f0 0x55c028727800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 4 client.25880 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c028655eb0 con 0x55c0287253f0 2023-07-14T11:12:35.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.216+0000 7f809effd700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287253f0 0x55c028727800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.217+0000 7f807effd700 10 client.25880 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:35.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.217+0000 7f809effd700 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287253f0 0x55c028727800 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9c1f096182c8065e server_cookie=a0089276a56fab19 in_seq=0 out_seq=0 2023-07-14T11:12:35.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.224+0000 7f807effd700 1 -- 172.21.15.40:0/4052997376 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c028655eb0 con 0x55c0287253f0 2023-07-14T11:12:35.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.224+0000 7f807effd700 10 client.25880 handle_command_reply: tid=0 2023-07-14T11:12:35.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.227+0000 7f80b95d7b80 1 client.25880 shutdown 2023-07-14T11:12:35.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.227+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 msgr2=0x55c028715560 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.227+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f8098009bc0 tx=0x7f8098011f90 comp rx=0 tx=0).stop 2023-07-14T11:12:35.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 shutdown_connections 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287253f0 0x55c028727800 crc :-1 s=READY pgs=56 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c028715ad0 0x55c028712f90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028715560 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/4052997376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870abe0 0x55c028712850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 >> 172.21.15.40:0/4052997376 conn(0x55c028700da0 msgr2=0x55c028706290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f807effd700 0 client.25880 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 shutdown_connections 2023-07-14T11:12:35.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.228+0000 7f80b95d7b80 1 -- 172.21.15.40:0/4052997376 wait complete. 2023-07-14T11:12:35.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.229+0000 7f80b95d7b80 20 client.25880 trim_cache size 0 max 16384 2023-07-14T11:12:35.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.244+0000 7f80b95d7b80 1 Processor -- start 2023-07-14T11:12:35.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.244+0000 7f80b95d7b80 1 -- start start 2023-07-14T11:12:35.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028720c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c028715ad0 0x55c02870db30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0287135c0 con 0x55c028715ad0 2023-07-14T11:12:35.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0287009a0 con 0x55c02870a2d0 2023-07-14T11:12:35.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c028713dd0 con 0x55c02870abe0 2023-07-14T11:12:35.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028720c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028720c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60622/0 (socket says 172.21.15.40:60622) 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809ffff700 1 -- 172.21.15.40:0/1844402785 learned_addr learned my addr 172.21.15.40:0/1844402785 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49498/0 (socket says 172.21.15.40:49498) 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.245+0000 7f809ffff700 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c028715ad0 0x55c02870db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 -- 172.21.15.40:0/1844402785 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 msgr2=0x55c028720c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028720c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 -- 172.21.15.40:0/1844402785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c028715ad0 msgr2=0x55c02870db30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c028715ad0 0x55c02870db30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 -- 172.21.15.40:0/1844402785 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c028473c60 con 0x55c02870abe0 2023-07-14T11:12:35.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f809effd700 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f8094009810 tx=0x7f809400f8e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=cccc7f3cd83c6132 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.246+0000 7f80867fc700 1 -- 172.21.15.40:0/1844402785 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8094027ad0 con 0x55c02870abe0 2023-07-14T11:12:35.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80867fc700 1 -- 172.21.15.40:0/1844402785 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f809401d040 con 0x55c02870abe0 2023-07-14T11:12:35.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80867fc700 1 -- 172.21.15.40:0/1844402785 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f809400b8e0 con 0x55c02870abe0 2023-07-14T11:12:35.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 -- 172.21.15.40:0/1844402785 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 msgr2=0x55c028715100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 crypto rx=0x7f8094009810 tx=0x7f809400f8e0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 -- 172.21.15.40:0/1844402785 shutdown_connections 2023-07-14T11:12:35.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c028715ad0 0x55c02870db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c02870a2d0 0x55c028720c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/1844402785 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c028715100 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 -- 172.21.15.40:0/1844402785 >> 172.21.15.40:0/1844402785 conn(0x55c028700da0 msgr2=0x55c028704e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 -- 172.21.15.40:0/1844402785 shutdown_connections 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.247+0000 7f80b95d7b80 1 -- 172.21.15.40:0/1844402785 wait complete. 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.248+0000 7f80b95d7b80 1 -- start start 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.248+0000 7f80b95d7b80 1 Processor -- start 2023-07-14T11:12:35.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c028719ae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 0x55c02871a560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0287782b0 con 0x55c02870a2d0 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c0287783f0 con 0x55c028715ad0 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c028778530 con 0x55c02870abe0 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f80b95d7b80 10 client.0 _ll_register_callbacks cb 0x7f80900ca8b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c028719ae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809effd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49500/0 (socket says 172.21.15.40:49500) 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c028719ae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39518/0 (socket says 172.21.15.40:39518) 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809effd700 1 -- 172.21.15.40:0/2951129215 learned_addr learned my addr 172.21.15.40:0/2951129215 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.249+0000 7f809ffff700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 0x55c02871a560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 -- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 msgr2=0x55c02871a560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 0x55c02871a560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 -- 172.21.15.40:0/2951129215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 msgr2=0x55c028719ae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:35.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c028719ae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 -- 172.21.15.40:0/2951129215 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f809400f700 con 0x55c02870abe0 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809ffff700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 0x55c02871a560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f809effd700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f8094009c40 tx=0x7f809400b2c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ff33da07cd92f639 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f807effd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f809400b760 con 0x55c02870abe0 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f809800f040 con 0x55c02870abe0 2023-07-14T11:12:35.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.250+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f809400b8c0 con 0x55c02870abe0 2023-07-14T11:12:35.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8094027ce0 con 0x55c02870abe0 2023-07-14T11:12:35.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 10 client.25873 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:35.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x55c0284b1250 con 0x55c02870abe0 2023-07-14T11:12:35.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f809800f040 con 0x55c02870abe0 2023-07-14T11:12:35.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8094009920 con 0x55c02870abe0 2023-07-14T11:12:35.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 10 client.25873 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:35.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 10 client.25873 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 20 client.25873 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287798c0 0x55c02877bcd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 4 client.25873 mds_command: new command op to 24569 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c028618050 con 0x55c0287798c0 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.251+0000 7f809ffff700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287798c0 0x55c02877bcd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f807effd700 10 client.25873 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:35.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f809ffff700 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287798c0 0x55c02877bcd0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=38e54e0549361bc0 server_cookie=82c0a0ee5e0df5f in_seq=0 out_seq=0 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f807effd700 1 -- 172.21.15.40:0/2951129215 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c028618050 con 0x55c0287798c0 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f807effd700 10 client.25873 handle_command_reply: tid=0 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 client.25873 shutdown 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 msgr2=0x55c02871a020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 crypto rx=0x7f8094009c40 tx=0x7f809400b2c0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 shutdown_connections 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c0287798c0 0x55c02877bcd0 crc :-1 s=READY pgs=57 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c028715ad0 0x55c02871a560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c02870a2d0 0x55c028719ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 --2- 172.21.15.40:0/2951129215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c02870abe0 0x55c02871a020 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f807effd700 0 client.25873 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.252+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 >> 172.21.15.40:0/2951129215 conn(0x55c028700da0 msgr2=0x55c0287216c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 shutdown_connections 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80b95d7b80 1 -- 172.21.15.40:0/2951129215 wait complete. 2023-07-14T11:12:35.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80b95d7b80 20 client.25873 trim_cache size 0 max 16384 2023-07-14T11:12:35.263 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:12:35.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 msgr2=0x7f80880a3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 0x7f80880a3880 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f80900caa80 tx=0x7f80900d0f00 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 msgr2=0x7f80a0197dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.253+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 crypto rx=0x7f809000eb30 tx=0x7f809000ee40 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 shutdown_connections 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f80880a13d0 0x7f80880a3880 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80a010c310 0x7f80a019c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80a00728b0 0x7f80a0197dd0 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 --2- 172.21.15.40:0/2082172661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f80a00731c0 0x7f80a0198310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:35.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 >> 172.21.15.40:0/2082172661 conn(0x7f80a00fb5f0 msgr2=0x7f80a010e3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:35.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 shutdown_connections 2023-07-14T11:12:35.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:35.254+0000 7f80a5bff700 1 -- 172.21.15.40:0/2082172661 wait complete. 2023-07-14T11:12:35.379 INFO:teuthology.orchestra.run.smithi040.stderr:4000000 of 50000000 tuples (8%) done (elapsed 9.87 s, remaining 113.52 s) 2023-07-14T11:12:35.649 INFO:teuthology.orchestra.run.smithi040.stderr:4100000 of 50000000 tuples (8%) done (elapsed 10.14 s, remaining 113.53 s) 2023-07-14T11:12:35.951 INFO:teuthology.orchestra.run.smithi040.stderr:4200000 of 50000000 tuples (8%) done (elapsed 10.44 s, remaining 113.88 s) 2023-07-14T11:12:36.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:35 smithi130 ceph-mon[129405]: pgmap v16: 129 pgs: 129 active+clean; 307 MiB data, 879 MiB used, 1.0 TiB / 1.0 TiB avail; 230 KiB/s rd, 46 MiB/s wr, 161 op/s 2023-07-14T11:12:36.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:35 smithi040 ceph-mon[97954]: pgmap v16: 129 pgs: 129 active+clean; 307 MiB data, 879 MiB used, 1.0 TiB / 1.0 TiB avail; 230 KiB/s rd, 46 MiB/s wr, 161 op/s 2023-07-14T11:12:36.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:35 smithi045 ceph-mon[128791]: pgmap v16: 129 pgs: 129 active+clean; 307 MiB data, 879 MiB used, 1.0 TiB / 1.0 TiB avail; 230 KiB/s rd, 46 MiB/s wr, 161 op/s 2023-07-14T11:12:36.116 INFO:teuthology.orchestra.run.smithi040.stderr:4300000 of 50000000 tuples (8%) done (elapsed 10.61 s, remaining 112.76 s) 2023-07-14T11:12:36.275 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2023-07-14T11:12:36.276 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2023-07-14T11:12:36.395 INFO:teuthology.orchestra.run.smithi040.stderr:4400000 of 50000000 tuples (8%) done (elapsed 10.89 s, remaining 112.84 s) 2023-07-14T11:12:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/1723410368 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 msgr2=0x7ffb980fff40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/1723410368 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb980fff40 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 crypto rx=0x7ffb8c009990 tx=0x7ffb8c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.506 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/1723410368 shutdown_connections 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/1723410368 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98100480 0x7ffb9810c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/1723410368 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98105ae0 0x7ffb98105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/1723410368 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb980fff40 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/1723410368 >> 172.21.15.40:0/1723410368 conn(0x7ffb980fb5f0 msgr2=0x7ffb980fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/1723410368 shutdown_connections 2023-07-14T11:12:36.507 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.504+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/1723410368 wait complete. 2023-07-14T11:12:36.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.505+0000 7ffb9d5ff700 1 Processor -- start 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 -- start start 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 0x7ffb98198340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 0x7ffb9819c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb9819cde0 con 0x7ffb98100480 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb9819cf50 con 0x7ffb98105ae0 2023-07-14T11:12:36.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb9d5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb9819d110 con 0x7ffb980ffad0 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49506/0 (socket says 172.21.15.40:49506) 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4028013913 learned_addr learned my addr 172.21.15.40:0/4028013913 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.506+0000 7ffb967fc700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 0x7ffb98198340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb977fe700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 0x7ffb9819c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4028013913 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 msgr2=0x7ffb9819c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 0x7ffb9819c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 msgr2=0x7ffb98198340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 0x7ffb98198340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4028013913 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb8c0095f0 con 0x7ffb980ffad0 2023-07-14T11:12:36.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb967fc700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 0x7ffb98198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb977fe700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 0x7ffb9819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.507+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ffb8800e9a0 tx=0x7ffb8800ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.508+0000 7ffb7ffff700 1 -- 172.21.15.40:0/4028013913 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb88009790 con 0x7ffb980ffad0 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.508+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb9819d3f0 con 0x7ffb980ffad0 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.508+0000 7ffb7ffff700 1 -- 172.21.15.40:0/4028013913 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb88010430 con 0x7ffb980ffad0 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.508+0000 7ffb7ffff700 1 -- 172.21.15.40:0/4028013913 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb88018680 con 0x7ffb980ffad0 2023-07-14T11:12:36.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.508+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb981a4c30 con 0x7ffb980ffad0 2023-07-14T11:12:36.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.513+0000 7ffb7ffff700 1 -- 172.21.15.40:0/4028013913 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7ffb880105a0 con 0x7ffb980ffad0 2023-07-14T11:12:36.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.514+0000 7ffb7ffff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 0x7ffb800a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 Processor -- start 2023-07-14T11:12:36.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffb7ffff700 1 -- 172.21.15.40:0/4028013913 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7ffb88014070 con 0x7ffb980ffad0 2023-07-14T11:12:36.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 -- start start 2023-07-14T11:12:36.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffb967fc700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 0x7ffb800a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fb5720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc0610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319ebbd30 con 0x557319fb5350 2023-07-14T11:12:36.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319f0eac0 con 0x557319fc0b50 2023-07-14T11:12:36.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.515+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319f07f50 con 0x557319fb5c60 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60672/0 (socket says 172.21.15.40:60672) 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 -- 172.21.15.40:0/3527564930 learned_addr learned my addr 172.21.15.40:0/3527564930 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fb5720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb967fc700 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc0610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 -- 172.21.15.40:0/3527564930 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 msgr2=0x557319fc0610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc0610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 -- 172.21.15.40:0/3527564930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 msgr2=0x557319fb5720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fb5720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb977fe700 1 -- 172.21.15.40:0/3527564930 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557319d35e60 con 0x557319fc0b50 2023-07-14T11:12:36.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.516+0000 7ffb967fc700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 0x7ffb800a3800 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ffb98199410 tx=0x7ffb8c004500 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.517+0000 7ffb977fe700 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7ffb90003eb0 tx=0x7ffb90009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4bfe512b9b0e0d9f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.517+0000 7ffb7d7fa700 1 -- 172.21.15.40:0/3527564930 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb900092d0 con 0x557319fc0b50 2023-07-14T11:12:36.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.517+0000 7ffb7d7fa700 1 -- 172.21.15.40:0/3527564930 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb9000f040 con 0x557319fc0b50 2023-07-14T11:12:36.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.517+0000 7ffb7d7fa700 1 -- 172.21.15.40:0/3527564930 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb9000e870 con 0x557319fc0b50 2023-07-14T11:12:36.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3527564930 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 msgr2=0x557319fc2f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7ffb90003eb0 tx=0x7ffb90009920 comp rx=0 tx=0).stop 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3527564930 shutdown_connections 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fc2f30 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fb5720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3527564930 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc0610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3527564930 >> 172.21.15.40:0/3527564930 conn(0x557319fabe20 msgr2=0x557319fb1720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3527564930 shutdown_connections 2023-07-14T11:12:36.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.518+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3527564930 wait complete. 2023-07-14T11:12:36.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.519+0000 7ffbb0f60b80 1 -- start start 2023-07-14T11:12:36.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.519+0000 7ffbb0f60b80 1 Processor -- start 2023-07-14T11:12:36.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fc0570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fbde20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319ea7b40 con 0x557319fb5350 2023-07-14T11:12:36.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319ea7840 con 0x557319fc0b50 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319d8b9a0 con 0x557319fb5c60 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffbb0f60b80 10 client.0 _ll_register_callbacks cb 0x557319ea57e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffb967fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fc0570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffb967fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49520/0 (socket says 172.21.15.40:49520) 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.520+0000 7ffb967fc700 1 -- 172.21.15.40:0/3165666943 learned_addr learned my addr 172.21.15.40:0/3165666943 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb977fe700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fbde20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb96ffd700 1 -- 172.21.15.40:0/3165666943 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 msgr2=0x557319fbd7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb96ffd700 1 -- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 msgr2=0x557319fbde20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fbde20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb96ffd700 1 -- 172.21.15.40:0/3165666943 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb90003b90 con 0x557319fb5350 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.521+0000 7ffb977fe700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fbde20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.522+0000 7ffb967fc700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.522+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fc0570 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7ffb880ca6d0 tx=0x7ffb880cac60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ab9d5a4ca0b3fcd6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.522+0000 7ffb7aff5700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb88099490 con 0x557319fb5350 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7ffb880995f0 con 0x557319fb5350 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557319e61570 con 0x557319fb5350 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7ffb880b8120 con 0x557319fb5350 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffbb0f60b80 10 client.15903 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:36.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.523+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x557319d1ece0 con 0x557319fb5350 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557319e61570 con 0x557319fb5350 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ffb88094cb0 con 0x557319fb5350 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 10 client.15903 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 10 client.15903 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 20 client.15903 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557319fd0440 0x557319fd2850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 4 client.15903 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:36.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.524+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557319f00f30 con 0x557319fd0440 2023-07-14T11:12:36.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.526+0000 7ffb967fc700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557319fd0440 0x557319fd2850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.526+0000 7ffb7aff5700 10 client.15903 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.526+0000 7ffb967fc700 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557319fd0440 0x557319fd2850 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=204f853388267fd4 server_cookie=4b5832abe96dfd9a in_seq=0 out_seq=0 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.527+0000 7ffb7aff5700 1 -- 172.21.15.40:0/3165666943 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x557319f00f30 con 0x557319fd0440 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.527+0000 7ffb7aff5700 10 client.15903 handle_command_reply: tid=0 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.527+0000 7ffbb0f60b80 1 client.15903 shutdown 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.527+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 msgr2=0x557319fc0570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.527+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fc0570 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 crypto rx=0x7ffb880ca6d0 tx=0x7ffb880cac60 comp rx=0 tx=0).stop 2023-07-14T11:12:36.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 shutdown_connections 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557319fd0440 0x557319fd2850 crc :-1 s=READY pgs=58 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fbde20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5350 0x557319fc0570 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffb7aff5700 0 client.15903 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/3165666943 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fbd7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 >> 172.21.15.40:0/3165666943 conn(0x557319fabe20 msgr2=0x557319fb1310 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 shutdown_connections 2023-07-14T11:12:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/3165666943 wait complete. 2023-07-14T11:12:36.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.528+0000 7ffbb0f60b80 20 client.15903 trim_cache size 0 max 16384 2023-07-14T11:12:36.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 Processor -- start 2023-07-14T11:12:36.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 -- start start 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc00a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fc0b50 0x557319fbd090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319fb65c0 con 0x557319fc0b50 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319fb6e50 con 0x557319fb5350 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.547+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319fb0510 con 0x557319fb5c60 2023-07-14T11:12:36.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb967fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc00a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60698/0 (socket says 172.21.15.40:60698) 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4115588508 learned_addr learned my addr 172.21.15.40:0/4115588508 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb977fe700 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fc0b50 0x557319fbd090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4115588508 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 msgr2=0x557319fc00a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc00a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4115588508 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fc0b50 msgr2=0x557319fbd090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fc0b50 0x557319fbd090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 -- 172.21.15.40:0/4115588508 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557319d5c2d0 con 0x557319fb5350 2023-07-14T11:12:36.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.548+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7ffb880cd360 tx=0x7ffb8809a560 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e34db852f9cd7c49 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.549+0000 7ffb7dffb700 1 -- 172.21.15.40:0/4115588508 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb88098fa0 con 0x557319fb5350 2023-07-14T11:12:36.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.549+0000 7ffb7dffb700 1 -- 172.21.15.40:0/4115588508 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb880962e0 con 0x557319fb5350 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.549+0000 7ffb7dffb700 1 -- 172.21.15.40:0/4115588508 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb880b83a0 con 0x557319fb5350 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/4115588508 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 msgr2=0x557319fc7520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7ffb880cd360 tx=0x7ffb8809a560 comp rx=0 tx=0).stop 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/4115588508 shutdown_connections 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fc0b50 0x557319fbd090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fb5350 0x557319fc7520 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/4115588508 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5c60 0x557319fc00a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/4115588508 >> 172.21.15.40:0/4115588508 conn(0x557319fabe20 msgr2=0x557319fcc6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/4115588508 shutdown_connections 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/4115588508 wait complete. 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.550+0000 7ffbb0f60b80 1 -- start start 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.551+0000 7ffbb0f60b80 1 Processor -- start 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.551+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 0x557319fc7720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.551+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 0x557319fb9ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.551+0000 7ffbb0f60b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319fbab70 con 0x557319fb5c60 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557319fbace0 con 0x557319fc0b50 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffbb0f60b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55731a024300 con 0x557319fb5350 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffbb0f60b80 10 client.0 _ll_register_callbacks cb 0x7ffb8809a040 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60704/0 (socket says 172.21.15.40:60704) 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 -- 172.21.15.40:0/1260776484 learned_addr learned my addr 172.21.15.40:0/1260776484 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.556 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 0x557319fc7720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.557 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb967fc700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 0x557319fb9ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 -- 172.21.15.40:0/1260776484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 msgr2=0x557319fc7720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 0x557319fc7720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 -- 172.21.15.40:0/1260776484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 msgr2=0x557319fb9ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 0x557319fb9ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 -- 172.21.15.40:0/1260776484 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x557319d1ece0 con 0x557319fc0b50 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb967fc700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 0x557319fb9ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:36.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 0x557319fc7720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.552+0000 7ffb977fe700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7ffb90003eb0 tx=0x7ffb90007730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=df4658c989c87de8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffb7aff5700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb900093e0 con 0x557319fc0b50 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7ffb90007d60 con 0x557319fc0b50 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ffb9000f040 con 0x557319fc0b50 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7ffb9000e4f0 con 0x557319fc0b50 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffbb0f60b80 10 client.25904 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:36.560 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x557319d34330 con 0x557319fc0b50 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.553+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb9000f040 con 0x557319fc0b50 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ffb900154e0 con 0x557319fc0b50 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 10 client.25904 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 10 client.25904 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 20 client.25904 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55731a025a90 0x55731a027ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 4 client.25904 mds_command: new command op to 24569 tid=0[{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2023-07-14T11:12:36.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x557319ec30d0 con 0x55731a025a90 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55731a025a90 0x55731a027ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffb96ffd700 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55731a025a90 0x55731a027ea0 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=358b8e9e64530f8a server_cookie=d3202150f8597d77 in_seq=0 out_seq=0 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.554+0000 7ffb7aff5700 10 client.25904 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.556+0000 7ffb7aff5700 1 -- 172.21.15.40:0/1260776484 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x557319ec30d0 con 0x55731a025a90 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.556+0000 7ffb7aff5700 10 client.25904 handle_command_reply: tid=0 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.556+0000 7ffbb0f60b80 1 client.25904 shutdown 2023-07-14T11:12:36.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 msgr2=0x557319fba460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7ffb90003eb0 tx=0x7ffb90007730 comp rx=0 tx=0).stop 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 shutdown_connections 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55731a025a90 0x55731a027ea0 crc :-1 s=READY pgs=59 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557319fc0b50 0x557319fba460 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557319fb5350 0x557319fc7720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 --2- 172.21.15.40:0/1260776484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557319fb5c60 0x557319fb9ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 >> 172.21.15.40:0/1260776484 conn(0x557319fabe20 msgr2=0x557319fbd090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffb7aff5700 0 client.25904 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 shutdown_connections 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 1 -- 172.21.15.40:0/1260776484 wait complete. 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.557+0000 7ffbb0f60b80 20 client.25904 trim_cache size 0 max 16384 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stdout: "return_code": 0, 2023-07-14T11:12:36.564 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_tag": "06a99009-5658-4960-8c49-c6b423d18ebd", 2023-07-14T11:12:36.565 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": "asynchronous" 2023-07-14T11:12:36.565 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:12:36.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 msgr2=0x7ffb800a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 0x7ffb800a3800 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7ffb98199410 tx=0x7ffb8c004500 comp rx=0 tx=0).stop 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 msgr2=0x7ffb98197e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 crypto rx=0x7ffb8800e9a0 tx=0x7ffb8800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 shutdown_connections 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb800a1350 0x7ffb800a3800 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb98105ae0 0x7ffb9819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.566 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb980ffad0 0x7ffb98197e00 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 --2- 172.21.15.40:0/4028013913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb98100480 0x7ffb98198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 >> 172.21.15.40:0/4028013913 conn(0x7ffb980fb5f0 msgr2=0x7ffb980fbb30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 shutdown_connections 2023-07-14T11:12:36.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.558+0000 7ffb9d5ff700 1 -- 172.21.15.40:0/4028013913 wait complete. 2023-07-14T11:12:36.582 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:12:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:4500000 of 50000000 tuples (9%) done (elapsed 11.24 s, remaining 113.66 s) 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 -- 172.21.15.40:0/3324457408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0728b0 msgr2=0x7fda2c072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 --2- 172.21.15.40:0/3324457408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0728b0 0x7fda2c072c80 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 crypto rx=0x7fda28009990 tx=0x7fda28009ca0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 -- 172.21.15.40:0/3324457408 shutdown_connections 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 --2- 172.21.15.40:0/3324457408 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c110ae0 0x7fda2c112ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 --2- 172.21.15.40:0/3324457408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0728b0 0x7fda2c072c80 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 --2- 172.21.15.40:0/3324457408 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c0731c0 0x7fda2c1105a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 -- 172.21.15.40:0/3324457408 >> 172.21.15.40:0/3324457408 conn(0x7fda2c0fb5f0 msgr2=0x7fda2c0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 -- 172.21.15.40:0/3324457408 shutdown_connections 2023-07-14T11:12:36.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.824+0000 7fda337ff700 1 -- 172.21.15.40:0/3324457408 wait complete. 2023-07-14T11:12:36.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.825+0000 7fda337ff700 1 Processor -- start 2023-07-14T11:12:36.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.825+0000 7fda337ff700 1 -- start start 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 0x7fda2c19c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c110ae0 0x7fda2c1a0c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda2c1a1290 con 0x7fda2c0731c0 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda2c1a1400 con 0x7fda2c0728b0 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda337ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fda2c1a15c0 con 0x7fda2c110ae0 2023-07-14T11:12:36.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda30d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 0x7fda2c19c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda30d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39616/0 (socket says 172.21.15.40:39616) 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda30d9a700 1 -- 172.21.15.40:0/3974267369 learned_addr learned my addr 172.21.15.40:0/3974267369 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c110ae0 0x7fda2c1a0c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 -- 172.21.15.40:0/3974267369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c110ae0 msgr2=0x7fda2c1a0c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c110ae0 0x7fda2c1a0c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 -- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 msgr2=0x7fda2c19c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda3159b700 1 -- 172.21.15.40:0/3974267369 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fda280095f0 con 0x7fda2c0728b0 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.826+0000 7fda30d9a700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda3159b700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 0x7fda2c19c2b0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fda28000c00 tx=0x7fda28005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda227fc700 1 -- 172.21.15.40:0/3974267369 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda2801d070 con 0x7fda2c0728b0 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fda2c1a1840 con 0x7fda2c0728b0 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda227fc700 1 -- 172.21.15.40:0/3974267369 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda28021410 con 0x7fda2c0728b0 2023-07-14T11:12:36.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda227fc700 1 -- 172.21.15.40:0/3974267369 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda28017650 con 0x7fda2c0728b0 2023-07-14T11:12:36.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.827+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda2c1a90e0 con 0x7fda2c0728b0 2023-07-14T11:12:36.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.830+0000 7fda227fc700 1 -- 172.21.15.40:0/3974267369 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 22) v1 ==== 135891+0+0 (secure 0 0 0) 0x7fda280217f0 con 0x7fda2c0728b0 2023-07-14T11:12:36.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.832+0000 7fda470eab80 1 Processor -- start 2023-07-14T11:12:36.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.832+0000 7fda470eab80 1 -- start start 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64ed6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f8590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda227fc700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 0x7fda180a3800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64faeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b63f3cb0 con 0x5572b64edbe0 2023-07-14T11:12:36.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b6446a40 con 0x5572b64ed2d0 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda470eab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b643fed0 con 0x5572b64f8ad0 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda227fc700 1 -- 172.21.15.40:0/3974267369 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fda280ce4b0 con 0x7fda2c0728b0 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda3159b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f8590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda3159b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f8590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:39620/0 (socket says 172.21.15.40:39620) 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda31d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64ed6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda3159b700 1 -- 172.21.15.40:0/3989653015 learned_addr learned my addr 172.21.15.40:0/3989653015 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.833+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64faeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda30d9a700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 0x7fda180a3800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 -- 172.21.15.40:0/3989653015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 msgr2=0x5572b64ed6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64ed6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 -- 172.21.15.40:0/3989653015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 msgr2=0x5572b64f8590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f8590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 -- 172.21.15.40:0/3989653015 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5572b626dde0 con 0x5572b64f8ad0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64faeb0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fda2c069aa0 tx=0x7fda2400b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c7520d809aa54de server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda30d9a700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 0x7fda180a3800 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fda2c19d8c0 tx=0x7fda1c009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda177fe700 1 -- 172.21.15.40:0/3989653015 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda2400e030 con 0x5572b64f8ad0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda177fe700 1 -- 172.21.15.40:0/3989653015 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda2400f040 con 0x5572b64f8ad0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.834+0000 7fda177fe700 1 -- 172.21.15.40:0/3989653015 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda2401a7f0 con 0x5572b64f8ad0 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 -- 172.21.15.40:0/3989653015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 msgr2=0x5572b64faeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64faeb0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fda2c069aa0 tx=0x7fda2400b040 comp rx=0 tx=0).stop 2023-07-14T11:12:36.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 -- 172.21.15.40:0/3989653015 shutdown_connections 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64faeb0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64ed6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 --2- 172.21.15.40:0/3989653015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f8590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 -- 172.21.15.40:0/3989653015 >> 172.21.15.40:0/3989653015 conn(0x5572b64e3da0 msgr2=0x5572b64e96a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 -- 172.21.15.40:0/3989653015 shutdown_connections 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.835+0000 7fda470eab80 1 -- 172.21.15.40:0/3989653015 wait complete. 2023-07-14T11:12:36.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.836+0000 7fda470eab80 1 -- start start 2023-07-14T11:12:36.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.836+0000 7fda470eab80 1 Processor -- start 2023-07-14T11:12:36.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f5d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64f6250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b63dfac0 con 0x5572b64edbe0 2023-07-14T11:12:36.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b63df7c0 con 0x5572b64ed2d0 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b62c3920 con 0x5572b64f8ad0 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda470eab80 10 client.0 _ll_register_callbacks cb 0x5572b63dd760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda30d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f5d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60752/0 (socket says 172.21.15.40:60752) 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 -- 172.21.15.40:0/3390113824 learned_addr learned my addr 172.21.15.40:0/3390113824 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda31d9c700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64f6250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 -- 172.21.15.40:0/3390113824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 msgr2=0x5572b64f6250 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64f6250 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 -- 172.21.15.40:0/3390113824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 msgr2=0x5572b64f5d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f5d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.837+0000 7fda3159b700 1 -- 172.21.15.40:0/3390113824 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fda24008510 con 0x5572b64ed2d0 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda3159b700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fda280cb7b0 tx=0x7fda2809b6b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f798f1fc560df899 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:12:36.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda280ce300 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fda2809b170 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5572b63994f0 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fda2809af40 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda470eab80 10 client.25916 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5572b6256c60 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5572b63994f0 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.838+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fda280a31c0 con 0x5572b64ed2d0 2023-07-14T11:12:36.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 10 client.25916 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 10 client.25916 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 20 client.25916 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b65081c0 0x5572b650a5d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 4 client.25916 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5572b6438eb0 con 0x5572b65081c0 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda30d9a700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b65081c0 0x5572b650a5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda30d9a700 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b65081c0 0x5572b650a5d0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c93032c251eb50ef server_cookie=80a317ec2464fdb6 in_seq=0 out_seq=0 2023-07-14T11:12:36.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.839+0000 7fda14ff9700 10 client.25916 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda14ff9700 1 -- 172.21.15.40:0/3390113824 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5572b6438eb0 con 0x5572b65081c0 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda14ff9700 10 client.25916 handle_command_reply: tid=0 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda470eab80 1 client.25916 shutdown 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 msgr2=0x5572b64f57d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fda280cb7b0 tx=0x7fda2809b6b0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 shutdown_connections 2023-07-14T11:12:36.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.840+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b65081c0 0x5572b650a5d0 crc :-1 s=READY pgs=60 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64f6250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f57d0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda14ff9700 0 client.25916 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 --2- 172.21.15.40:0/3390113824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64edbe0 0x5572b64f5d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 >> 172.21.15.40:0/3390113824 conn(0x5572b64e3da0 msgr2=0x5572b64e9230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 shutdown_connections 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 1 -- 172.21.15.40:0/3390113824 wait complete. 2023-07-14T11:12:36.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.841+0000 7fda470eab80 20 client.25916 trim_cache size 0 max 16384 2023-07-14T11:12:36.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.858+0000 7fda470eab80 1 Processor -- start 2023-07-14T11:12:36.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 -- start start 2023-07-14T11:12:36.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64edbe0 0x5572b64f4690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 0x5572b64f2a40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b64eee30 con 0x5572b64f8ad0 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b64f8860 con 0x5572b64ed2d0 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda470eab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b64ef1c0 con 0x5572b64edbe0 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda3159b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda3159b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60754/0 (socket says 172.21.15.40:60754) 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda3159b700 1 -- 172.21.15.40:0/2994055873 learned_addr learned my addr 172.21.15.40:0/2994055873 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda31d9c700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 0x5572b64f2a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.859+0000 7fda30d9a700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64edbe0 0x5572b64f4690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 -- 172.21.15.40:0/2994055873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64edbe0 msgr2=0x5572b64f4690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64edbe0 0x5572b64f4690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 -- 172.21.15.40:0/2994055873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 msgr2=0x5572b64f2a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 0x5572b64f2a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 -- 172.21.15.40:0/2994055873 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5572b6294250 con 0x5572b64ed2d0 2023-07-14T11:12:36.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda31d9c700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 0x5572b64f2a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:12:36.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda3159b700 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fda2800b2e0 tx=0x7fda28099720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c7d57cd9029c7f32 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda17fff700 1 -- 172.21.15.40:0/2994055873 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda280ce300 con 0x5572b64ed2d0 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda17fff700 1 -- 172.21.15.40:0/2994055873 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda28021e90 con 0x5572b64ed2d0 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.860+0000 7fda17fff700 1 -- 172.21.15.40:0/2994055873 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda280cb4b0 con 0x5572b64ed2d0 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 -- 172.21.15.40:0/2994055873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 msgr2=0x5572b64f6080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7fda2800b2e0 tx=0x7fda28099720 comp rx=0 tx=0).stop 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 -- 172.21.15.40:0/2994055873 shutdown_connections 2023-07-14T11:12:36.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64f8ad0 0x5572b64f2a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64ed2d0 0x5572b64f6080 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 --2- 172.21.15.40:0/2994055873 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64edbe0 0x5572b64f4690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 -- 172.21.15.40:0/2994055873 >> 172.21.15.40:0/2994055873 conn(0x5572b64e3da0 msgr2=0x5572b64f3f80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 -- 172.21.15.40:0/2994055873 shutdown_connections 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.861+0000 7fda470eab80 1 -- 172.21.15.40:0/2994055873 wait complete. 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.862+0000 7fda470eab80 1 -- start start 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.862+0000 7fda470eab80 1 Processor -- start 2023-07-14T11:12:36.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64ed2d0 0x5572b64ff6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 0x5572b64fd990 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b64fe7a0 con 0x5572b64ed2d0 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b655c090 con 0x5572b64edbe0 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5572b655c1d0 con 0x5572b64f8ad0 2023-07-14T11:12:36.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda470eab80 10 client.0 _ll_register_callbacks cb 0x7fda280057b0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:12:36.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda31d9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda3159b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64ed2d0 0x5572b64ff6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda31d9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49574/0 (socket says 172.21.15.40:49574) 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.863+0000 7fda31d9c700 1 -- 172.21.15.40:0/1214778237 learned_addr learned my addr 172.21.15.40:0/1214778237 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda30d9a700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 0x5572b64fd990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 -- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 msgr2=0x5572b64fd990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 0x5572b64fd990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 -- 172.21.15.40:0/1214778237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64ed2d0 msgr2=0x5572b64ff6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:12:36.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64ed2d0 0x5572b64ff6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 -- 172.21.15.40:0/1214778237 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5572b6256c60 con 0x5572b64f8ad0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda30d9a700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 0x5572b64fd990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda31d9c700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fda2400b010 tx=0x7fda24018890 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b4a7352b488d6792 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.864+0000 7fda14ff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fda2400e030 con 0x5572b64f8ad0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fda2400f040 con 0x5572b64f8ad0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fda240090f0 con 0x5572b64f8ad0 2023-07-14T11:12:36.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=27) v2 ==== 24+0+0 (secure 0 0 0) 0x7fda24003e90 con 0x5572b64f8ad0 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 10 client.25909 fetch_fsmap learned FSMap version 27 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=27,monmap=4+}) v3 -- 0x5572b626c2b0 con 0x5572b64f8ad0 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fda2400f040 con 0x5572b64f8ad0 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 27) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fda2402c020 con 0x5572b64f8ad0 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 10 client.25909 fetch_fsmap finished waiting for FSMap version 27 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 10 client.25909 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 20 client.25909 populate_metadata read hostname 'smithi040' 2023-07-14T11:12:36.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b655d6e0 0x5572b655faf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 4 client.25909 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.865+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5572b63fb050 con 0x5572b655d6e0 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda3159b700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b655d6e0 0x5572b655faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda3159b700 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b655d6e0 0x5572b655faf0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8c14004e39f25bc6 server_cookie=6bbb9cd6257e8e9c in_seq=0 out_seq=0 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda14ff9700 10 client.25909 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda14ff9700 1 -- 172.21.15.40:0/1214778237 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+259 (crc 0 0 0) 0x5572b63fb050 con 0x5572b655d6e0 2023-07-14T11:12:36.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda14ff9700 10 client.25909 handle_command_reply: tid=0 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda470eab80 1 client.25909 shutdown 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 msgr2=0x5572b64fe0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fda2400b010 tx=0x7fda24018890 comp rx=0 tx=0).stop 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 shutdown_connections 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.866+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5572b655d6e0 0x5572b655faf0 crc :-1 s=READY pgs=61 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5572b64f8ad0 0x5572b64fe0d0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5572b64ed2d0 0x5572b64ff6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 --2- 172.21.15.40:0/1214778237 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5572b64edbe0 0x5572b64fd990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 >> 172.21.15.40:0/1214778237 conn(0x5572b64e3da0 msgr2=0x5572b64efc70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda14ff9700 0 client.25909 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 shutdown_connections 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 1 -- 172.21.15.40:0/1214778237 wait complete. 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda470eab80 20 client.25909 trim_cache size 0 max 16384 2023-07-14T11:12:36.875 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "IDLE (waiting for more scrubs)", 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "06a99009-5658-4960-8c49-c6b423d18ebd": { 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "06a99009-5658-4960-8c49-c6b423d18ebd", 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:12:36.876 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:12:36.877 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:12:36.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 msgr2=0x7fda180a3800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 0x7fda180a3800 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fda2c19d8c0 tx=0x7fda1c009260 comp rx=0 tx=0).stop 2023-07-14T11:12:36.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 msgr2=0x7fda2c19c2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.867+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 0x7fda2c19c2b0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7fda28000c00 tx=0x7fda28005740 comp rx=0 tx=0).stop 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 shutdown_connections 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fda180a1350 0x7fda180a3800 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fda2c110ae0 0x7fda2c1a0c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fda2c0728b0 0x7fda2c19c2b0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 --2- 172.21.15.40:0/3974267369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fda2c0731c0 0x7fda2c19c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:12:36.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 >> 172.21.15.40:0/3974267369 conn(0x7fda2c0fb5f0 msgr2=0x7fda2c0fc250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:12:36.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 shutdown_connections 2023-07-14T11:12:36.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:12:36.868+0000 7fda337ff700 1 -- 172.21.15.40:0/3974267369 wait complete. 2023-07-14T11:12:36.883 INFO:teuthology.orchestra.run.smithi040.stderr:4600000 of 50000000 tuples (9%) done (elapsed 11.38 s, remaining 112.27 s) 2023-07-14T11:12:36.887 INFO:tasks.cephfs.filesystem:scrub status for tag:06a99009-5658-4960-8c49-c6b423d18ebd - {'path': '/', 'tag': '06a99009-5658-4960-8c49-c6b423d18ebd', 'options': 'recursive,force'} 2023-07-14T11:12:37.243 INFO:teuthology.orchestra.run.smithi040.stderr:4700000 of 50000000 tuples (9%) done (elapsed 11.74 s, remaining 113.12 s) 2023-07-14T11:12:37.552 INFO:teuthology.orchestra.run.smithi040.stderr:4800000 of 50000000 tuples (9%) done (elapsed 12.04 s, remaining 113.42 s) 2023-07-14T11:12:37.667 INFO:teuthology.orchestra.run.smithi040.stderr:4900000 of 50000000 tuples (9%) done (elapsed 12.16 s, remaining 111.92 s) 2023-07-14T11:12:38.017 INFO:teuthology.orchestra.run.smithi040.stderr:5000000 of 50000000 tuples (10%) done (elapsed 12.51 s, remaining 112.59 s) 2023-07-14T11:12:38.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:37 smithi130 ceph-mon[129405]: pgmap v17: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 62 MiB/s wr, 152 op/s 2023-07-14T11:12:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:37 smithi130 ceph-mon[129405]: scrub queued for path: / 2023-07-14T11:12:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:37 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:37 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:12:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:37 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:37 smithi040 ceph-mon[97954]: pgmap v17: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 62 MiB/s wr, 152 op/s 2023-07-14T11:12:38.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:37 smithi040 ceph-mon[97954]: scrub queued for path: / 2023-07-14T11:12:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:37 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:37 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:12:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:37 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:37 smithi045 ceph-mon[128791]: pgmap v17: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 232 KiB/s rd, 62 MiB/s wr, 152 op/s 2023-07-14T11:12:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:37 smithi045 ceph-mon[128791]: scrub queued for path: / 2023-07-14T11:12:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:37 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:37 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:12:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:37 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:12:38.133 INFO:teuthology.orchestra.run.smithi040.stderr:5100000 of 50000000 tuples (10%) done (elapsed 12.63 s, remaining 111.16 s) 2023-07-14T11:12:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:5200000 of 50000000 tuples (10%) done (elapsed 12.94 s, remaining 111.44 s) 2023-07-14T11:12:38.871 INFO:teuthology.orchestra.run.smithi040.stderr:5300000 of 50000000 tuples (10%) done (elapsed 13.36 s, remaining 112.71 s) 2023-07-14T11:12:38.981 INFO:teuthology.orchestra.run.smithi040.stderr:5400000 of 50000000 tuples (10%) done (elapsed 13.47 s, remaining 111.29 s) 2023-07-14T11:12:39.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:38 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:12:39.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:38 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:12:39.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:38 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:12:39.285 INFO:teuthology.orchestra.run.smithi040.stderr:5500000 of 50000000 tuples (11%) done (elapsed 13.78 s, remaining 111.48 s) 2023-07-14T11:12:39.445 INFO:teuthology.orchestra.run.smithi040.stderr:5600000 of 50000000 tuples (11%) done (elapsed 13.94 s, remaining 110.51 s) 2023-07-14T11:12:39.727 INFO:teuthology.orchestra.run.smithi040.stderr:5700000 of 50000000 tuples (11%) done (elapsed 14.22 s, remaining 110.52 s) 2023-07-14T11:12:40.023 INFO:teuthology.orchestra.run.smithi040.stderr:5800000 of 50000000 tuples (11%) done (elapsed 14.52 s, remaining 110.62 s) 2023-07-14T11:12:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:39 smithi130 ceph-mon[129405]: pgmap v18: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 173 KiB/s rd, 61 MiB/s wr, 47 op/s 2023-07-14T11:12:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:39 smithi040 ceph-mon[97954]: pgmap v18: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 173 KiB/s rd, 61 MiB/s wr, 47 op/s 2023-07-14T11:12:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:39 smithi045 ceph-mon[128791]: pgmap v18: 129 pgs: 129 active+clean; 411 MiB data, 1.1 GiB used, 1.0 TiB / 1.0 TiB avail; 173 KiB/s rd, 61 MiB/s wr, 47 op/s 2023-07-14T11:12:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:40.221 INFO:teuthology.orchestra.run.smithi040.stderr:5900000 of 50000000 tuples (11%) done (elapsed 14.71 s, remaining 109.98 s) 2023-07-14T11:12:41.168 INFO:teuthology.orchestra.run.smithi040.stderr:6000000 of 50000000 tuples (12%) done (elapsed 15.66 s, remaining 114.85 s) 2023-07-14T11:12:41.931 INFO:teuthology.orchestra.run.smithi040.stderr:6100000 of 50000000 tuples (12%) done (elapsed 16.42 s, remaining 118.20 s) 2023-07-14T11:12:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:41 smithi130 ceph-mon[129405]: pgmap v19: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 202 KiB/s rd, 95 MiB/s wr, 75 op/s 2023-07-14T11:12:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:41 smithi040 ceph-mon[97954]: pgmap v19: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 202 KiB/s rd, 95 MiB/s wr, 75 op/s 2023-07-14T11:12:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:41 smithi045 ceph-mon[128791]: pgmap v19: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 202 KiB/s rd, 95 MiB/s wr, 75 op/s 2023-07-14T11:12:42.069 INFO:teuthology.orchestra.run.smithi040.stderr:6200000 of 50000000 tuples (12%) done (elapsed 16.56 s, remaining 117.00 s) 2023-07-14T11:12:42.724 INFO:teuthology.orchestra.run.smithi040.stderr:6300000 of 50000000 tuples (12%) done (elapsed 17.22 s, remaining 119.43 s) 2023-07-14T11:12:43.025 INFO:teuthology.orchestra.run.smithi040.stderr:6400000 of 50000000 tuples (12%) done (elapsed 17.52 s, remaining 119.34 s) 2023-07-14T11:12:43.174 INFO:teuthology.orchestra.run.smithi040.stderr:6500000 of 50000000 tuples (13%) done (elapsed 17.67 s, remaining 118.23 s) 2023-07-14T11:12:43.468 INFO:teuthology.orchestra.run.smithi040.stderr:6600000 of 50000000 tuples (13%) done (elapsed 17.96 s, remaining 118.10 s) 2023-07-14T11:12:43.637 INFO:teuthology.orchestra.run.smithi040.stderr:6700000 of 50000000 tuples (13%) done (elapsed 18.13 s, remaining 117.16 s) 2023-07-14T11:12:43.923 INFO:teuthology.orchestra.run.smithi040.stderr:6800000 of 50000000 tuples (13%) done (elapsed 18.42 s, remaining 117.00 s) 2023-07-14T11:12:44.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:43 smithi130 ceph-mon[129405]: pgmap v20: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 66 MiB/s wr, 51 op/s 2023-07-14T11:12:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:43 smithi040 ceph-mon[97954]: pgmap v20: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 66 MiB/s wr, 51 op/s 2023-07-14T11:12:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:43 smithi045 ceph-mon[128791]: pgmap v20: 129 pgs: 129 active+clean; 608 MiB data, 1.5 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 66 MiB/s wr, 51 op/s 2023-07-14T11:12:44.214 INFO:teuthology.orchestra.run.smithi040.stderr:6900000 of 50000000 tuples (13%) done (elapsed 18.71 s, remaining 116.85 s) 2023-07-14T11:12:44.393 INFO:teuthology.orchestra.run.smithi040.stderr:7000000 of 50000000 tuples (14%) done (elapsed 18.89 s, remaining 116.01 s) 2023-07-14T11:12:44.677 INFO:teuthology.orchestra.run.smithi040.stderr:7100000 of 50000000 tuples (14%) done (elapsed 19.17 s, remaining 115.83 s) 2023-07-14T11:12:44.996 INFO:teuthology.orchestra.run.smithi040.stderr:7200000 of 50000000 tuples (14%) done (elapsed 19.49 s, remaining 115.85 s) 2023-07-14T11:12:45.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:44 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:45.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:44 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:45.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:44 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:45.182 INFO:teuthology.orchestra.run.smithi040.stderr:7300000 of 50000000 tuples (14%) done (elapsed 19.67 s, remaining 115.08 s) 2023-07-14T11:12:45.459 INFO:teuthology.orchestra.run.smithi040.stderr:7400000 of 50000000 tuples (14%) done (elapsed 19.95 s, remaining 114.86 s) 2023-07-14T11:12:45.748 INFO:teuthology.orchestra.run.smithi040.stderr:7500000 of 50000000 tuples (15%) done (elapsed 20.24 s, remaining 114.70 s) 2023-07-14T11:12:45.942 INFO:teuthology.orchestra.run.smithi040.stderr:7600000 of 50000000 tuples (15%) done (elapsed 20.44 s, remaining 114.01 s) 2023-07-14T11:12:46.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:45 smithi130 ceph-mon[129405]: pgmap v21: 129 pgs: 129 active+clean; 896 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 92 MiB/s wr, 61 op/s 2023-07-14T11:12:46.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:45 smithi040 ceph-mon[97954]: pgmap v21: 129 pgs: 129 active+clean; 896 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 92 MiB/s wr, 61 op/s 2023-07-14T11:12:46.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:45 smithi045 ceph-mon[128791]: pgmap v21: 129 pgs: 129 active+clean; 896 MiB data, 2.0 GiB used, 1.0 TiB / 1.0 TiB avail; 68 KiB/s rd, 92 MiB/s wr, 61 op/s 2023-07-14T11:12:46.199 INFO:teuthology.orchestra.run.smithi040.stderr:7700000 of 50000000 tuples (15%) done (elapsed 20.69 s, remaining 113.67 s) 2023-07-14T11:12:46.505 INFO:teuthology.orchestra.run.smithi040.stderr:7800000 of 50000000 tuples (15%) done (elapsed 21.00 s, remaining 113.60 s) 2023-07-14T11:12:46.678 INFO:teuthology.orchestra.run.smithi040.stderr:7900000 of 50000000 tuples (15%) done (elapsed 21.17 s, remaining 112.82 s) 2023-07-14T11:12:46.947 INFO:teuthology.orchestra.run.smithi040.stderr:8000000 of 50000000 tuples (16%) done (elapsed 21.44 s, remaining 112.56 s) 2023-07-14T11:12:47.237 INFO:teuthology.orchestra.run.smithi040.stderr:8100000 of 50000000 tuples (16%) done (elapsed 21.73 s, remaining 112.41 s) 2023-07-14T11:12:47.354 INFO:teuthology.orchestra.run.smithi040.stderr:8200000 of 50000000 tuples (16%) done (elapsed 21.85 s, remaining 111.36 s) 2023-07-14T11:12:47.769 INFO:teuthology.orchestra.run.smithi040.stderr:8300000 of 50000000 tuples (16%) done (elapsed 22.26 s, remaining 111.85 s) 2023-07-14T11:12:48.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:47 smithi130 ceph-mon[129405]: pgmap v22: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 139 MiB/s wr, 74 op/s 2023-07-14T11:12:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:47 smithi040 ceph-mon[97954]: pgmap v22: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 139 MiB/s wr, 74 op/s 2023-07-14T11:12:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:47 smithi045 ceph-mon[128791]: pgmap v22: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 67 KiB/s rd, 139 MiB/s wr, 74 op/s 2023-07-14T11:12:48.089 INFO:teuthology.orchestra.run.smithi040.stderr:8400000 of 50000000 tuples (16%) done (elapsed 22.58 s, remaining 111.83 s) 2023-07-14T11:12:48.220 INFO:teuthology.orchestra.run.smithi040.stderr:8500000 of 50000000 tuples (17%) done (elapsed 22.71 s, remaining 110.89 s) 2023-07-14T11:12:48.576 INFO:teuthology.orchestra.run.smithi040.stderr:8600000 of 50000000 tuples (17%) done (elapsed 23.07 s, remaining 111.05 s) 2023-07-14T11:12:48.694 INFO:teuthology.orchestra.run.smithi040.stderr:8700000 of 50000000 tuples (17%) done (elapsed 23.19 s, remaining 110.07 s) 2023-07-14T11:12:49.023 INFO:teuthology.orchestra.run.smithi040.stderr:8800000 of 50000000 tuples (17%) done (elapsed 23.52 s, remaining 110.10 s) 2023-07-14T11:12:49.396 INFO:teuthology.orchestra.run.smithi040.stderr:8900000 of 50000000 tuples (17%) done (elapsed 23.89 s, remaining 110.32 s) 2023-07-14T11:12:49.505 INFO:teuthology.orchestra.run.smithi040.stderr:9000000 of 50000000 tuples (18%) done (elapsed 24.00 s, remaining 109.32 s) 2023-07-14T11:12:49.874 INFO:teuthology.orchestra.run.smithi040.stderr:9100000 of 50000000 tuples (18%) done (elapsed 24.37 s, remaining 109.52 s) 2023-07-14T11:12:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:49 smithi130 ceph-mon[129405]: pgmap v23: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 121 MiB/s wr, 67 op/s 2023-07-14T11:12:50.074 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:49 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:50.075 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:49 smithi045 ceph-mon[128791]: pgmap v23: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 121 MiB/s wr, 67 op/s 2023-07-14T11:12:50.075 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:49 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:50.075 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:49 smithi040 ceph-mon[97954]: pgmap v23: 129 pgs: 129 active+clean; 1.5 GiB data, 3.3 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 121 MiB/s wr, 67 op/s 2023-07-14T11:12:50.075 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:49 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:12:50.275 INFO:teuthology.orchestra.run.smithi040.stderr:9200000 of 50000000 tuples (18%) done (elapsed 24.77 s, remaining 109.84 s) 2023-07-14T11:12:50.385 INFO:teuthology.orchestra.run.smithi040.stderr:9300000 of 50000000 tuples (18%) done (elapsed 24.88 s, remaining 108.87 s) 2023-07-14T11:12:50.706 INFO:teuthology.orchestra.run.smithi040.stderr:9400000 of 50000000 tuples (18%) done (elapsed 25.20 s, remaining 108.84 s) 2023-07-14T11:12:50.915 INFO:teuthology.orchestra.run.smithi040.stderr:9500000 of 50000000 tuples (19%) done (elapsed 25.41 s, remaining 108.32 s) 2023-07-14T11:12:51.187 INFO:teuthology.orchestra.run.smithi040.stderr:9600000 of 50000000 tuples (19%) done (elapsed 25.68 s, remaining 108.07 s) 2023-07-14T11:12:51.491 INFO:teuthology.orchestra.run.smithi040.stderr:9700000 of 50000000 tuples (19%) done (elapsed 25.98 s, remaining 107.95 s) 2023-07-14T11:12:51.753 INFO:teuthology.orchestra.run.smithi040.stderr:9800000 of 50000000 tuples (19%) done (elapsed 26.25 s, remaining 107.66 s) 2023-07-14T11:12:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:51 smithi130 ceph-mon[129405]: pgmap v24: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 154 MiB/s wr, 93 op/s 2023-07-14T11:12:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:51 smithi040 ceph-mon[97954]: pgmap v24: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 154 MiB/s wr, 93 op/s 2023-07-14T11:12:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:51 smithi045 ceph-mon[128791]: pgmap v24: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 154 MiB/s wr, 93 op/s 2023-07-14T11:12:52.282 INFO:teuthology.orchestra.run.smithi040.stderr:9900000 of 50000000 tuples (19%) done (elapsed 26.77 s, remaining 108.45 s) 2023-07-14T11:12:52.776 INFO:teuthology.orchestra.run.smithi040.stderr:10000000 of 50000000 tuples (20%) done (elapsed 27.27 s, remaining 109.08 s) 2023-07-14T11:12:53.082 INFO:teuthology.orchestra.run.smithi040.stderr:10100000 of 50000000 tuples (20%) done (elapsed 27.58 s, remaining 108.94 s) 2023-07-14T11:12:53.461 INFO:teuthology.orchestra.run.smithi040.stderr:10200000 of 50000000 tuples (20%) done (elapsed 27.95 s, remaining 109.07 s) 2023-07-14T11:12:53.686 INFO:teuthology.orchestra.run.smithi040.stderr:10300000 of 50000000 tuples (20%) done (elapsed 28.18 s, remaining 108.61 s) 2023-07-14T11:12:53.899 INFO:teuthology.orchestra.run.smithi040.stderr:10400000 of 50000000 tuples (20%) done (elapsed 28.39 s, remaining 108.11 s) 2023-07-14T11:12:54.032 INFO:teuthology.orchestra.run.smithi040.stderr:10500000 of 50000000 tuples (21%) done (elapsed 28.53 s, remaining 107.31 s) 2023-07-14T11:12:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:53 smithi130 ceph-mon[129405]: pgmap v25: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 120 MiB/s wr, 65 op/s 2023-07-14T11:12:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:53 smithi040 ceph-mon[97954]: pgmap v25: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 120 MiB/s wr, 65 op/s 2023-07-14T11:12:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:53 smithi045 ceph-mon[128791]: pgmap v25: 129 pgs: 129 active+clean; 1.7 GiB data, 3.7 GiB used, 1.0 TiB / 1.0 TiB avail; 72 KiB/s rd, 120 MiB/s wr, 65 op/s 2023-07-14T11:12:54.216 INFO:teuthology.orchestra.run.smithi040.stderr:10600000 of 50000000 tuples (21%) done (elapsed 28.71 s, remaining 106.71 s) 2023-07-14T11:12:54.431 INFO:teuthology.orchestra.run.smithi040.stderr:10700000 of 50000000 tuples (21%) done (elapsed 28.92 s, remaining 106.23 s) 2023-07-14T11:12:54.585 INFO:teuthology.orchestra.run.smithi040.stderr:10800000 of 50000000 tuples (21%) done (elapsed 29.08 s, remaining 105.54 s) 2023-07-14T11:12:54.758 INFO:teuthology.orchestra.run.smithi040.stderr:10900000 of 50000000 tuples (21%) done (elapsed 29.25 s, remaining 104.93 s) 2023-07-14T11:12:54.969 INFO:teuthology.orchestra.run.smithi040.stderr:11000000 of 50000000 tuples (22%) done (elapsed 29.46 s, remaining 104.46 s) 2023-07-14T11:12:55.146 INFO:teuthology.orchestra.run.smithi040.stderr:11100000 of 50000000 tuples (22%) done (elapsed 29.64 s, remaining 103.87 s) 2023-07-14T11:12:55.342 INFO:teuthology.orchestra.run.smithi040.stderr:11200000 of 50000000 tuples (22%) done (elapsed 29.84 s, remaining 103.36 s) 2023-07-14T11:12:55.468 INFO:teuthology.orchestra.run.smithi040.stderr:11300000 of 50000000 tuples (22%) done (elapsed 29.96 s, remaining 102.61 s) 2023-07-14T11:12:55.674 INFO:teuthology.orchestra.run.smithi040.stderr:11400000 of 50000000 tuples (22%) done (elapsed 30.17 s, remaining 102.14 s) 2023-07-14T11:12:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:55 smithi130 ceph-mon[129405]: pgmap v26: 129 pgs: 129 active+clean; 2.0 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 151 MiB/s wr, 76 op/s 2023-07-14T11:12:55.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:55 smithi040 ceph-mon[97954]: pgmap v26: 129 pgs: 129 active+clean; 2.0 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 151 MiB/s wr, 76 op/s 2023-07-14T11:12:55.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:55 smithi045 ceph-mon[128791]: pgmap v26: 129 pgs: 129 active+clean; 2.0 GiB data, 4.3 GiB used, 1.0 TiB / 1.0 TiB avail; 73 KiB/s rd, 151 MiB/s wr, 76 op/s 2023-07-14T11:12:55.882 INFO:teuthology.orchestra.run.smithi040.stderr:11500000 of 50000000 tuples (23%) done (elapsed 30.37 s, remaining 101.69 s) 2023-07-14T11:12:56.037 INFO:teuthology.orchestra.run.smithi040.stderr:11600000 of 50000000 tuples (23%) done (elapsed 30.53 s, remaining 101.06 s) 2023-07-14T11:12:56.186 INFO:teuthology.orchestra.run.smithi040.stderr:11700000 of 50000000 tuples (23%) done (elapsed 30.68 s, remaining 100.43 s) 2023-07-14T11:12:56.418 INFO:teuthology.orchestra.run.smithi040.stderr:11800000 of 50000000 tuples (23%) done (elapsed 30.91 s, remaining 100.07 s) 2023-07-14T11:12:56.599 INFO:teuthology.orchestra.run.smithi040.stderr:11900000 of 50000000 tuples (23%) done (elapsed 31.09 s, remaining 99.55 s) 2023-07-14T11:12:56.780 INFO:teuthology.orchestra.run.smithi040.stderr:12000000 of 50000000 tuples (24%) done (elapsed 31.27 s, remaining 99.03 s) 2023-07-14T11:12:56.990 INFO:teuthology.orchestra.run.smithi040.stderr:12100000 of 50000000 tuples (24%) done (elapsed 31.48 s, remaining 98.61 s) 2023-07-14T11:12:57.116 INFO:teuthology.orchestra.run.smithi040.stderr:12200000 of 50000000 tuples (24%) done (elapsed 31.61 s, remaining 97.94 s) 2023-07-14T11:12:57.305 INFO:teuthology.orchestra.run.smithi040.stderr:12300000 of 50000000 tuples (24%) done (elapsed 31.80 s, remaining 97.46 s) 2023-07-14T11:12:57.520 INFO:teuthology.orchestra.run.smithi040.stderr:12400000 of 50000000 tuples (24%) done (elapsed 32.01 s, remaining 97.07 s) 2023-07-14T11:12:57.733 INFO:teuthology.orchestra.run.smithi040.stderr:12500000 of 50000000 tuples (25%) done (elapsed 32.23 s, remaining 96.68 s) 2023-07-14T11:12:57.847 INFO:teuthology.orchestra.run.smithi040.stderr:12600000 of 50000000 tuples (25%) done (elapsed 32.34 s, remaining 95.99 s) 2023-07-14T11:12:58.034 INFO:teuthology.orchestra.run.smithi040.stderr:12700000 of 50000000 tuples (25%) done (elapsed 32.53 s, remaining 95.53 s) 2023-07-14T11:12:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:57 smithi130 ceph-mon[129405]: pgmap v27: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 163 MiB/s wr, 94 op/s 2023-07-14T11:12:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:57 smithi040 ceph-mon[97954]: pgmap v27: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 163 MiB/s wr, 94 op/s 2023-07-14T11:12:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:57 smithi045 ceph-mon[128791]: pgmap v27: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 87 KiB/s rd, 163 MiB/s wr, 94 op/s 2023-07-14T11:12:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:12800000 of 50000000 tuples (25%) done (elapsed 32.73 s, remaining 95.11 s) 2023-07-14T11:12:58.819 INFO:teuthology.orchestra.run.smithi040.stderr:12900000 of 50000000 tuples (25%) done (elapsed 33.31 s, remaining 95.80 s) 2023-07-14T11:12:59.133 INFO:teuthology.orchestra.run.smithi040.stderr:13000000 of 50000000 tuples (26%) done (elapsed 33.63 s, remaining 95.70 s) 2023-07-14T11:12:59.359 INFO:teuthology.orchestra.run.smithi040.stderr:13100000 of 50000000 tuples (26%) done (elapsed 33.85 s, remaining 95.35 s) 2023-07-14T11:12:59.561 INFO:teuthology.orchestra.run.smithi040.stderr:13200000 of 50000000 tuples (26%) done (elapsed 34.05 s, remaining 94.94 s) 2023-07-14T11:12:59.852 INFO:teuthology.orchestra.run.smithi040.stderr:13300000 of 50000000 tuples (26%) done (elapsed 34.34 s, remaining 94.77 s) 2023-07-14T11:13:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:12:59 smithi130 ceph-mon[129405]: pgmap v28: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 102 MiB/s wr, 65 op/s 2023-07-14T11:13:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:12:59 smithi040 ceph-mon[97954]: pgmap v28: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 102 MiB/s wr, 65 op/s 2023-07-14T11:13:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:12:59 smithi045 ceph-mon[128791]: pgmap v28: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 63 KiB/s rd, 102 MiB/s wr, 65 op/s 2023-07-14T11:13:00.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:13:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:13:00.158+0000 7fc6d787b700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:13:00.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:13:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:13:00.158+0000 7fc6d787b700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:13:00.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:13:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:13:00.158+0000 7fc6d787b700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:13:00.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:13:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:13:00.158+0000 7fc6d787b700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:13:00.547 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:13:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:13:00.158+0000 7fc6d787b700 -1 client.0 error registering admin socket command: (17) File exists 2023-07-14T11:13:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:01 smithi130 ceph-mon[129405]: pgmap v29: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 123 MiB/s wr, 108 op/s 2023-07-14T11:13:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:01 smithi130 ceph-mon[129405]: mgrmap e23: x(active, since 52s), standbys: z, y 2023-07-14T11:13:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:01 smithi040 ceph-mon[97954]: pgmap v29: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 123 MiB/s wr, 108 op/s 2023-07-14T11:13:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:01 smithi040 ceph-mon[97954]: mgrmap e23: x(active, since 52s), standbys: z, y 2023-07-14T11:13:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:01 smithi045 ceph-mon[128791]: pgmap v29: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 76 KiB/s rd, 123 MiB/s wr, 108 op/s 2023-07-14T11:13:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:01 smithi045 ceph-mon[128791]: mgrmap e23: x(active, since 52s), standbys: z, y 2023-07-14T11:13:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:03 smithi130 ceph-mon[129405]: pgmap v30: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 91 MiB/s wr, 82 op/s 2023-07-14T11:13:04.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:03 smithi040 ceph-mon[97954]: pgmap v30: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 91 MiB/s wr, 82 op/s 2023-07-14T11:13:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:03 smithi045 ceph-mon[128791]: pgmap v30: 129 pgs: 129 active+clean; 2.3 GiB data, 4.9 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 91 MiB/s wr, 82 op/s 2023-07-14T11:13:05.033 INFO:teuthology.orchestra.run.smithi040.stderr:13400000 of 50000000 tuples (26%) done (elapsed 39.53 s, remaining 107.96 s) 2023-07-14T11:13:05.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:05 smithi130 ceph-mon[129405]: pgmap v31: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 92 MiB/s wr, 155 op/s 2023-07-14T11:13:05.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:05 smithi040 ceph-mon[97954]: pgmap v31: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 92 MiB/s wr, 155 op/s 2023-07-14T11:13:05.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:05 smithi045 ceph-mon[128791]: pgmap v31: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 92 MiB/s wr, 155 op/s 2023-07-14T11:13:06.888 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:13:07.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 -- 172.21.15.40:0/637261184 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80731c0 msgr2=0x7f3dc810bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 --2- 172.21.15.40:0/637261184 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80731c0 0x7f3dc810bdd0 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f3db40098f0 tx=0x7f3db4009c00 comp rx=0 tx=0).stop 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 -- 172.21.15.40:0/637261184 shutdown_connections 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 --2- 172.21.15.40:0/637261184 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc810e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 --2- 172.21.15.40:0/637261184 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80728b0 0x7f3dc8072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.167+0000 7f3dcefff700 1 --2- 172.21.15.40:0/637261184 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80731c0 0x7f3dc810bdd0 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.168+0000 7f3dcefff700 1 -- 172.21.15.40:0/637261184 >> 172.21.15.40:0/637261184 conn(0x7f3dc80fb5f0 msgr2=0x7f3dc80fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.168+0000 7f3dcefff700 1 -- 172.21.15.40:0/637261184 shutdown_connections 2023-07-14T11:13:07.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.168+0000 7f3dcefff700 1 -- 172.21.15.40:0/637261184 wait complete. 2023-07-14T11:13:07.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.169+0000 7f3dcefff700 1 Processor -- start 2023-07-14T11:13:07.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 -- start start 2023-07-14T11:13:07.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 0x7f3dc8197e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80731c0 0x7f3dc8198360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dc819ce00 con 0x7f3dc80731c0 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dc819cf70 con 0x7f3dc80728b0 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dcefff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dc819d130 con 0x7f3dc810c310 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.170+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 0x7f3dc8197e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 0x7f3dc8197e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49242/0 (socket says 172.21.15.40:49242) 2023-07-14T11:13:07.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dccd9b700 1 -- 172.21.15.40:0/390875709 learned_addr learned my addr 172.21.15.40:0/390875709 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80731c0 0x7f3dc8198360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 -- 172.21.15.40:0/390875709 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 msgr2=0x7f3dc8197e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 0x7f3dc8197e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 -- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80731c0 msgr2=0x7f3dc8198360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80731c0 0x7f3dc8198360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.171+0000 7f3dcd59c700 1 -- 172.21.15.40:0/390875709 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3db4009520 con 0x7f3dc810c310 2023-07-14T11:13:07.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc819c7c0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f3dc000efc0 tx=0x7f3dc000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dc5ffb700 1 -- 172.21.15.40:0/390875709 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dc0018070 con 0x7f3dc810c310 2023-07-14T11:13:07.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3dc819d3b0 con 0x7f3dc810c310 2023-07-14T11:13:07.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dc5ffb700 1 -- 172.21.15.40:0/390875709 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3dc000f040 con 0x7f3dc810c310 2023-07-14T11:13:07.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dc5ffb700 1 -- 172.21.15.40:0/390875709 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dc000ccb0 con 0x7f3dc810c310 2023-07-14T11:13:07.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.172+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dc80ffb40 con 0x7f3dc810c310 2023-07-14T11:13:07.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.176+0000 7f3dc5ffb700 1 -- 172.21.15.40:0/390875709 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f3dc0003a00 con 0x7f3dc810c310 2023-07-14T11:13:07.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.177+0000 7f3de286cb80 1 Processor -- start 2023-07-14T11:13:07.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.177+0000 7f3dc5ffb700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 0x7f3db80a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3dc5ffb700 1 -- 172.21.15.40:0/390875709 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3dc00ccc60 con 0x7f3dc810c310 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 -- start start 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 0x7f3db80a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b6029116a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b602911be0 0x55b60291c590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602817cb0 con 0x55b6029112d0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b60286aa40 con 0x55b602911be0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.178+0000 7f3de286cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602863ed0 con 0x55b60291cad0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dc7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b602911be0 0x55b60291c590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50598/0 (socket says 172.21.15.40:50598) 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 -- 172.21.15.40:0/2543155823 learned_addr learned my addr 172.21.15.40:0/2543155823 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b6029116a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 -- 172.21.15.40:0/2543155823 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b602911be0 msgr2=0x55b60291c590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b602911be0 0x55b60291c590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 -- 172.21.15.40:0/2543155823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 msgr2=0x55b6029116a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b6029116a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 -- 172.21.15.40:0/2543155823 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b602691de0 con 0x55b60291cad0 2023-07-14T11:13:07.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f3dc000c6f0 tx=0x7f3dc0099440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=829f300c379c58ff server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3db2ffd700 1 -- 172.21.15.40:0/2543155823 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dc00a1bc0 con 0x55b60291cad0 2023-07-14T11:13:07.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3db2ffd700 1 -- 172.21.15.40:0/2543155823 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3dc000ce10 con 0x55b60291cad0 2023-07-14T11:13:07.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.179+0000 7f3db2ffd700 1 -- 172.21.15.40:0/2543155823 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dc0099cc0 con 0x55b60291cad0 2023-07-14T11:13:07.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 -- 172.21.15.40:0/2543155823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 msgr2=0x55b60291eeb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 0x7f3db80a3910 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f3db4005e90 tx=0x7f3db4005dc0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 crypto rx=0x7f3dc000c6f0 tx=0x7f3dc0099440 comp rx=0 tx=0).stop 2023-07-14T11:13:07.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 -- 172.21.15.40:0/2543155823 shutdown_connections 2023-07-14T11:13:07.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b60291cad0 0x55b60291eeb0 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b6029116a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 --2- 172.21.15.40:0/2543155823 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b602911be0 0x55b60291c590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 -- 172.21.15.40:0/2543155823 >> 172.21.15.40:0/2543155823 conn(0x55b602907da0 msgr2=0x55b60290d6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.180+0000 7f3de286cb80 1 -- 172.21.15.40:0/2543155823 shutdown_connections 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.181+0000 7f3de286cb80 1 -- 172.21.15.40:0/2543155823 wait complete. 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.181+0000 7f3de286cb80 1 -- start start 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.181+0000 7f3de286cb80 1 Processor -- start 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.182+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b6029197f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602919f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602803ac0 con 0x55b6029112d0 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6028037c0 con 0x55b60291cad0 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6026e7920 con 0x55b602911be0 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3de286cb80 10 client.0 _ll_register_callbacks cb 0x55b602801760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:07.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:44916/0 (socket says 172.21.15.40:44916) 2023-07-14T11:13:07.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3dccd9b700 1 -- 172.21.15.40:0/1542819589 learned_addr learned my addr 172.21.15.40:0/1542819589 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b6029197f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.183+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602919f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 -- 172.21.15.40:0/1542819589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 msgr2=0x55b6029197f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b6029197f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 -- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 msgr2=0x55b602919f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602919f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 -- 172.21.15.40:0/1542819589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dc80ffb40 con 0x55b6029112d0 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f3dbc012740 tx=0x7f3dbc012a50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=54b56859d0d79581 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.184+0000 7f3da7fff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dbc017070 con 0x55b6029112d0 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3dbc00f4e0 con 0x55b6029112d0 2023-07-14T11:13:07.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b6027bd4f0 con 0x55b6029112d0 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3dbc0122e0 con 0x55b6029112d0 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3de286cb80 10 client.15945 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55b60267ac60 con 0x55b6029112d0 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b6027bd4f0 con 0x55b6029112d0 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.185+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f3dbc0108c0 con 0x55b6029112d0 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 10 client.15945 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 10 client.15945 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:07.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 20 client.15945 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b60292c4f0 0x55b60292e900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 4 client.15945 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b60285ceb0 con 0x55b60292c4f0 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.186+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b60292c4f0 0x55b60292e900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.187+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b60292c4f0 0x55b60292e900 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ab963b1ff9b13472 server_cookie=6d3eaf2da7c4a3f4 in_seq=0 out_seq=0 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.187+0000 7f3da7fff700 10 client.15945 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.187+0000 7f3da7fff700 1 -- 172.21.15.40:0/1542819589 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55b60285ceb0 con 0x55b60292c4f0 2023-07-14T11:13:07.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3da7fff700 10 client.15945 handle_command_reply: tid=0 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 client.15945 shutdown 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 msgr2=0x55b60291c500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 crypto rx=0x7f3dbc012740 tx=0x7f3dbc012a50 comp rx=0 tx=0).stop 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 shutdown_connections 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b60292c4f0 0x55b60292e900 crc :-1 s=READY pgs=63 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602919f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c500 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 --2- 172.21.15.40:0/1542819589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b6029197f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 >> 172.21.15.40:0/1542819589 conn(0x55b602907da0 msgr2=0x55b60290d290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3da7fff700 0 client.15945 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 shutdown_connections 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.188+0000 7f3de286cb80 1 -- 172.21.15.40:0/1542819589 wait complete. 2023-07-14T11:13:07.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.189+0000 7f3de286cb80 20 client.15945 trim_cache size 0 max 16384 2023-07-14T11:13:07.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 Processor -- start 2023-07-14T11:13:07.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 -- start start 2023-07-14T11:13:07.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b602914bd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602918fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b60291ad60 con 0x55b6029112d0 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602904fa0 con 0x55b60291cad0 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.214+0000 7f3de286cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602907970 con 0x55b602911be0 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:44922/0 (socket says 172.21.15.40:44922) 2023-07-14T11:13:07.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 -- 172.21.15.40:0/569941687 learned_addr learned my addr 172.21.15.40:0/569941687 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b602914bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602918fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 -- 172.21.15.40:0/569941687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 msgr2=0x55b602914bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b602914bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 -- 172.21.15.40:0/569941687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 msgr2=0x55b602918fb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602918fb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.215+0000 7f3dccd9b700 1 -- 172.21.15.40:0/569941687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b6029095e0 con 0x55b6029112d0 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.216+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f3dbc011fa0 tx=0x7f3dbc011fd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3d4c17fca846284f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.216+0000 7f3db37fe700 1 -- 172.21.15.40:0/569941687 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dbc017070 con 0x55b6029112d0 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.216+0000 7f3db37fe700 1 -- 172.21.15.40:0/569941687 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3dbc018040 con 0x55b6029112d0 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.216+0000 7f3db37fe700 1 -- 172.21.15.40:0/569941687 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3dbc003680 con 0x55b6029112d0 2023-07-14T11:13:07.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 -- 172.21.15.40:0/569941687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 msgr2=0x55b60291c0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 crypto rx=0x7f3dbc011fa0 tx=0x7f3dbc011fd0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 -- 172.21.15.40:0/569941687 shutdown_connections 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602918fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60291c0a0 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 --2- 172.21.15.40:0/569941687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b602914bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 -- 172.21.15.40:0/569941687 >> 172.21.15.40:0/569941687 conn(0x55b602907da0 msgr2=0x55b60292e500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 -- 172.21.15.40:0/569941687 shutdown_connections 2023-07-14T11:13:07.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.217+0000 7f3de286cb80 1 -- 172.21.15.40:0/569941687 wait complete. 2023-07-14T11:13:07.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.218+0000 7f3de286cb80 1 -- start start 2023-07-14T11:13:07.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.218+0000 7f3de286cb80 1 Processor -- start 2023-07-14T11:13:07.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60297b390 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602920f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b60297c1e0 con 0x55b6029112d0 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b602921440 con 0x55b60291cad0 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b6029215d0 con 0x55b602911be0 2023-07-14T11:13:07.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3de286cb80 10 client.0 _ll_register_callbacks cb 0x7f3dbc009df0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3dc7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3dc7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50632/0 (socket says 172.21.15.40:50632) 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.219+0000 7f3dc7fff700 1 -- 172.21.15.40:0/606471842 learned_addr learned my addr 172.21.15.40:0/606471842 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 -- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 msgr2=0x55b602920f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602920f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602920f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 -- 172.21.15.40:0/606471842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 msgr2=0x55b60297b390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60297b390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 -- 172.21.15.40:0/606471842 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55b60267ac60 con 0x55b602911be0 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dcd59c700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602920f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3dc7fff700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f3db40098f0 tx=0x7f3db4009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2f6d886842779ba8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3da7fff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.220+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3db402e040 con 0x55b602911be0 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3db400b560 con 0x55b602911be0 2023-07-14T11:13:07.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3dc000ce10 con 0x55b602911be0 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3db4034620 con 0x55b602911be0 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3de286cb80 10 client.25933 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55b6026b8250 con 0x55b602911be0 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.221+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3dc000ce10 con 0x55b602911be0 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f3db4003680 con 0x55b602911be0 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 10 client.25933 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 10 client.25933 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:07.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 20 client.25933 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:07.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b602992210 0x55b602994620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 4 client.25933 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:13:07.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55b60281f050 con 0x55b602992210 2023-07-14T11:13:07.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.222+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b602992210 0x55b602994620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3dccd9b700 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b602992210 0x55b602994620 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ea8205d65295f48a server_cookie=5b286380049bbaf in_seq=0 out_seq=0 2023-07-14T11:13:07.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3da7fff700 10 client.25933 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:07.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3da7fff700 1 -- 172.21.15.40:0/606471842 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x55b60281f050 con 0x55b602992210 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3da7fff700 10 client.25933 handle_command_reply: tid=0 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3de286cb80 1 client.25933 shutdown 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 msgr2=0x55b60297b8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.223+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 crypto rx=0x7f3db40098f0 tx=0x7f3db4009260 comp rx=0 tx=0).stop 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 shutdown_connections 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b602992210 0x55b602994620 crc :-1 s=READY pgs=64 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b60291cad0 0x55b602920f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3da7fff700 0 client.25933 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b6029112d0 0x55b60297b390 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 --2- 172.21.15.40:0/606471842 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b602911be0 0x55b60297b8d0 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 >> 172.21.15.40:0/606471842 conn(0x55b602907da0 msgr2=0x55b602914bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 shutdown_connections 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.224+0000 7f3de286cb80 1 -- 172.21.15.40:0/606471842 wait complete. 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3de286cb80 20 client.25933 trim_cache size 0 max 16384 2023-07-14T11:13:07.231 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:13:07.232 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "no active scrubs running", 2023-07-14T11:13:07.232 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": {} 2023-07-14T11:13:07.232 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 msgr2=0x7f3db80a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 0x7f3db80a3910 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f3db4005e90 tx=0x7f3db4005dc0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 msgr2=0x7f3dc819c7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc819c7c0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 crypto rx=0x7f3dc000efc0 tx=0x7f3dc000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 shutdown_connections 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3db80a1460 0x7f3db80a3910 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dc810c310 0x7f3dc819c7c0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dc80728b0 0x7f3dc8197e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 --2- 172.21.15.40:0/390875709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dc80731c0 0x7f3dc8198360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.225+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 >> 172.21.15.40:0/390875709 conn(0x7f3dc80fb5f0 msgr2=0x7f3dc810e3f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.226+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 shutdown_connections 2023-07-14T11:13:07.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.226+0000 7f3dcefff700 1 -- 172.21.15.40:0/390875709 wait complete. 2023-07-14T11:13:07.251 INFO:tasks.cephfs.filesystem:all active scrubs completed 2023-07-14T11:13:07.252 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:13:07.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 -- 172.21.15.40:0/801856759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00ff3a0 msgr2=0x7f8fd0103870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/801856759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd0103870 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 crypto rx=0x7f8fc00098f0 tx=0x7f8fc0009c00 comp rx=0 tx=0).stop 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 -- 172.21.15.40:0/801856759 shutdown_connections 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/801856759 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 0x7f8fd01042e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/801856759 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00fea90 0x7f8fd00fee60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/801856759 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd0103870 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.526+0000 7f8fd7fff700 1 -- 172.21.15.40:0/801856759 >> 172.21.15.40:0/801856759 conn(0x7f8fd00fa490 msgr2=0x7f8fd00fc8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.527+0000 7f8fd7fff700 1 -- 172.21.15.40:0/801856759 shutdown_connections 2023-07-14T11:13:07.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.527+0000 7f8fd7fff700 1 -- 172.21.15.40:0/801856759 wait complete. 2023-07-14T11:13:07.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 Processor -- start 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 -- start start 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd019c710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 0x7f8fd01a0b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8fd01a11b0 con 0x7f8fd0103e40 2023-07-14T11:13:07.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8fd01a1320 con 0x7f8fd00ff3a0 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.528+0000 7f8fd7fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8fd01a14e0 con 0x7f8fd00fea90 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50650/0 (socket says 172.21.15.40:50650) 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 -- 172.21.15.40:0/916549396 learned_addr learned my addr 172.21.15.40:0/916549396 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd659c700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 0x7f8fd01a0b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd559a700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd019c710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 -- 172.21.15.40:0/916549396 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00ff3a0 msgr2=0x7f8fd019c710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd019c710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 -- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 msgr2=0x7f8fd01a0b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 0x7f8fd01a0b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.529+0000 7f8fd5d9b700 1 -- 172.21.15.40:0/916549396 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8fc8009d00 con 0x7f8fd00fea90 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fd5d9b700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8fcc00e9a0 tx=0x7f8fcc00ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fcc009790 con 0x7f8fd00fea90 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8fcc00cbd0 con 0x7f8fd00fea90 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8fc0009520 con 0x7f8fd00fea90 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8fcc018680 con 0x7f8fd00fea90 2023-07-14T11:13:07.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.530+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8fd01a19b0 con 0x7f8fd00fea90 2023-07-14T11:13:07.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.532+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8fd0067300 con 0x7f8fd00fea90 2023-07-14T11:13:07.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.534+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f8fcc00cd40 con 0x7f8fd00fea90 2023-07-14T11:13:07.537 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.536+0000 7f8fc6ffd700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 0x7f8fbc0a3960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:07.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.536+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8fcc014070 con 0x7f8fd00fea90 2023-07-14T11:13:07.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.537+0000 7f8fd559a700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 0x7f8fbc0a3960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:07.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.537+0000 7f8fd559a700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 0x7f8fbc0a3960 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f8fc000c010 tx=0x7f8fc0009f50 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:07.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.538+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8fcc096470 con 0x7f8fd00fea90 2023-07-14T11:13:07.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.896+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8fd019d520 con 0x7f8fd00fea90 2023-07-14T11:13:07.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.897+0000 7f8fc6ffd700 1 -- 172.21.15.40:0/916549396 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 30 v30) v1 ==== 94+0+10897 (secure 0 0 0) 0x7f8fcc099a90 con 0x7f8fd00fea90 2023-07-14T11:13:07.899 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:13:07.899 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":30,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":30,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:12:48.628040+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[1,2,3,4],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:13:07.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 msgr2=0x7f8fbc0a3960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 0x7f8fbc0a3960 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f8fc000c010 tx=0x7f8fc0009f50 comp rx=0 tx=0).stop 2023-07-14T11:13:07.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 msgr2=0x7f8fd019c1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:07.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 crypto rx=0x7f8fcc00e9a0 tx=0x7f8fcc00ed60 comp rx=0 tx=0).stop 2023-07-14T11:13:07.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 shutdown_connections 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8fbc0a14b0 0x7f8fbc0a3960 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8fd0103e40 0x7f8fd01a0b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8fd00fea90 0x7f8fd019c1d0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 --2- 172.21.15.40:0/916549396 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8fd00ff3a0 0x7f8fd019c710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 >> 172.21.15.40:0/916549396 conn(0x7f8fd00fa490 msgr2=0x7f8fd0101920 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 shutdown_connections 2023-07-14T11:13:07.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:07.900+0000 7f8fd7fff700 1 -- 172.21.15.40:0/916549396 wait complete. 2023-07-14T11:13:07.904 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 30 2023-07-14T11:13:07.923 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:07 smithi040 ceph-mon[97954]: pgmap v32: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 238 op/s 2023-07-14T11:13:07.926 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:4 damage ls 2023-07-14T11:13:08.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:07 smithi130 ceph-mon[129405]: pgmap v32: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 238 op/s 2023-07-14T11:13:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:07 smithi045 ceph-mon[128791]: pgmap v32: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 63 MiB/s wr, 238 op/s 2023-07-14T11:13:08.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 -- 172.21.15.40:0/4115714067 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0fea90 msgr2=0x7f16dc0fee60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 --2- 172.21.15.40:0/4115714067 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0fea90 0x7f16dc0fee60 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f16cc0098f0 tx=0x7f16cc009c00 comp rx=0 tx=0).stop 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 -- 172.21.15.40:0/4115714067 shutdown_connections 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 --2- 172.21.15.40:0/4115714067 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc103e40 0x7f16dc1042e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 --2- 172.21.15.40:0/4115714067 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0fea90 0x7f16dc0fee60 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 --2- 172.21.15.40:0/4115714067 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc103870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 -- 172.21.15.40:0/4115714067 >> 172.21.15.40:0/4115714067 conn(0x7f16dc0fa4b0 msgr2=0x7f16dc0fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 -- 172.21.15.40:0/4115714067 shutdown_connections 2023-07-14T11:13:08.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.229+0000 7f16e1bff700 1 -- 172.21.15.40:0/4115714067 wait complete. 2023-07-14T11:13:08.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.230+0000 7f16e1bff700 1 Processor -- start 2023-07-14T11:13:08.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 -- start start 2023-07-14T11:13:08.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc0fea90 0x7f16dc19c300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 0x7f16dc1a0ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc1a12e0 con 0x7f16dc0fea90 2023-07-14T11:13:08.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc1a1450 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.231+0000 7f16e1bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16dc1a1610 con 0x7f16dc103e40 2023-07-14T11:13:08.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49294/0 (socket says 172.21.15.40:49294) 2023-07-14T11:13:08.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 -- 172.21.15.40:0/2987900526 learned_addr learned my addr 172.21.15.40:0/2987900526 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc0fea90 0x7f16dc19c300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16dbfff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 0x7f16dc1a0ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 -- 172.21.15.40:0/2987900526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 msgr2=0x7f16dc1a0ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 0x7f16dc1a0ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 -- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc0fea90 msgr2=0x7f16dc19c300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc0fea90 0x7f16dc19c300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 -- 172.21.15.40:0/2987900526 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16cc009520 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16dbfff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 0x7f16dc1a0ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.232+0000 7f16daffd700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f16d000e9c0 tx=0x7f16d000ecd0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.233+0000 7f16d8ff9700 1 -- 172.21.15.40:0/2987900526 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d000cbd0 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.233+0000 7f16d8ff9700 1 -- 172.21.15.40:0/2987900526 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f16d000eeb0 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.233+0000 7f16d8ff9700 1 -- 172.21.15.40:0/2987900526 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d0005580 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.233+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16dc1a18f0 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.233+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16dc1a9130 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.236+0000 7f16d8ff9700 1 -- 172.21.15.40:0/2987900526 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f16d0004750 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.239+0000 7f16d8ff9700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 0x7f16c40a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.239+0000 7f16d8ff9700 1 -- 172.21.15.40:0/2987900526 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f16d0014070 con 0x7f16dc0ff3a0 2023-07-14T11:13:08.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.239+0000 7f16f5546b80 1 Processor -- start 2023-07-14T11:13:08.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 0x7f16c40a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 -- start start 2023-07-14T11:13:08.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 0x563591bd66a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 0x563591be3eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591adccb0 con 0x563591bd62d0 2023-07-14T11:13:08.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591b2fa40 con 0x563591bd6be0 2023-07-14T11:13:08.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.240+0000 7f16f5546b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591b28ed0 con 0x563591be1ad0 2023-07-14T11:13:08.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16dbfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 0x563591be3eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49306/0 (socket says 172.21.15.40:49306) 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 -- 172.21.15.40:0/3671417784 learned_addr learned my addr 172.21.15.40:0/3671417784 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16db7fe700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 0x563591bd66a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 -- 172.21.15.40:0/3671417784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 msgr2=0x563591be3eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 0x563591be3eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 -- 172.21.15.40:0/3671417784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 msgr2=0x563591bd66a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 0x563591bd66a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 -- 172.21.15.40:0/3671417784 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563591956de0 con 0x563591bd6be0 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16dbfff700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 0x563591be3eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16db7fe700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 0x563591bd66a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 0x7f16c40a3910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f16cc005e90 tx=0x7f16cc019040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.241+0000 7f16daffd700 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f16d0096c20 tx=0x7f16d000cd30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8ce8a36b46e15a02 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.242+0000 7f16c1ffb700 1 -- 172.21.15.40:0/3671417784 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d009e1e0 con 0x563591bd6be0 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.242+0000 7f16c1ffb700 1 -- 172.21.15.40:0/3671417784 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f16d00a2600 con 0x563591bd6be0 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.242+0000 7f16c1ffb700 1 -- 172.21.15.40:0/3671417784 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d009a670 con 0x563591bd6be0 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 -- 172.21.15.40:0/3671417784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 msgr2=0x563591be1590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f16d0096c20 tx=0x7f16d000cd30 comp rx=0 tx=0).stop 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 -- 172.21.15.40:0/3671417784 shutdown_connections 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591be1ad0 0x563591be3eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd62d0 0x563591bd66a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 --2- 172.21.15.40:0/3671417784 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd6be0 0x563591be1590 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 -- 172.21.15.40:0/3671417784 >> 172.21.15.40:0/3671417784 conn(0x563591bccda0 msgr2=0x563591bd26a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 -- 172.21.15.40:0/3671417784 shutdown_connections 2023-07-14T11:13:08.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.243+0000 7f16f5546b80 1 -- 172.21.15.40:0/3671417784 wait complete. 2023-07-14T11:13:08.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.244+0000 7f16f5546b80 1 -- start start 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.244+0000 7f16f5546b80 1 Processor -- start 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 0x563591bde8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591bdedf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591ac8ac0 con 0x563591be1ad0 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591ac87c0 con 0x563591bd62d0 2023-07-14T11:13:08.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635919ac920 con 0x563591bd6be0 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16f5546b80 10 client.0 _ll_register_callbacks cb 0x563591ac6760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.245+0000 7f16dbfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 0x563591bde8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16dbfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:44992/0 (socket says 172.21.15.40:44992) 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16dbfff700 1 -- 172.21.15.40:0/4091923408 learned_addr learned my addr 172.21.15.40:0/4091923408 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16daffd700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591bdedf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 -- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 msgr2=0x563591bdedf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591bdedf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 -- 172.21.15.40:0/4091923408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 msgr2=0x563591bdf330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16db7fe700 1 -- 172.21.15.40:0/4091923408 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16d40090d0 con 0x563591bd62d0 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.246+0000 7f16dbfff700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16db7fe700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 0x563591bde8b0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009810 tx=0x7f16cc009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6028748e7f5805ec server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16baffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:08.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16cc00ba50 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x563591a824f0 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f16cc005040 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f16cc00bbb0 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16f5546b80 10 client.25958 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.247+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x7f16dc1a9130 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x563591a824f0 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f16cc036070 con 0x563591bd62d0 2023-07-14T11:13:08.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 10 client.25958 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 10 client.25958 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:13:08.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 20 client.25958 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591bf12c0 0x563591bf36a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 4 client.25958 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:08.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x563591b21eb0 con 0x563591bf12c0 2023-07-14T11:13:08.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.248+0000 7f16daffd700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591bf12c0 0x563591bf36a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.249+0000 7f16daffd700 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591bf12c0 0x563591bf36a0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=6542401eabf7d46d server_cookie=2f84f3f245444fa4 in_seq=0 out_seq=0 2023-07-14T11:13:08.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.249+0000 7f16baffd700 10 client.25958 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16baffd700 1 -- 172.21.15.40:0/4091923408 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x563591b21eb0 con 0x563591bf12c0 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16baffd700 10 client.25958 handle_command_reply: tid=0 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 client.25958 shutdown 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 msgr2=0x563591bde8b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 0x563591bde8b0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009810 tx=0x7f16cc009260 comp rx=0 tx=0).stop 2023-07-14T11:13:08.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 shutdown_connections 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591bf12c0 0x563591bf36a0 crc :-1 s=READY pgs=36 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bd62d0 0x563591bde8b0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 --2- 172.21.15.40:0/4091923408 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591bdedf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 >> 172.21.15.40:0/4091923408 conn(0x563591bccda0 msgr2=0x563591bd2230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16baffd700 0 client.25958 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 shutdown_connections 2023-07-14T11:13:08.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.250+0000 7f16f5546b80 1 -- 172.21.15.40:0/4091923408 wait complete. 2023-07-14T11:13:08.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.251+0000 7f16f5546b80 20 client.25958 trim_cache size 0 max 16384 2023-07-14T11:13:08.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.269+0000 7f16f5546b80 1 Processor -- start 2023-07-14T11:13:08.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.269+0000 7f16f5546b80 1 -- start start 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf190 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bda2e0 0x563591bdbb60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591bc9fa0 con 0x563591be1ad0 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591bcc970 con 0x563591bda2e0 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16f5546b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591bd68f0 con 0x563591bd6be0 2023-07-14T11:13:08.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50700/0 (socket says 172.21.15.40:50700) 2023-07-14T11:13:08.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16db7fe700 1 -- 172.21.15.40:0/2701690723 learned_addr learned my addr 172.21.15.40:0/2701690723 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.270+0000 7f16dbfff700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bda2e0 0x563591bdbb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16daffd700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 -- 172.21.15.40:0/2701690723 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bda2e0 msgr2=0x563591bdbb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bda2e0 0x563591bdbb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 -- 172.21.15.40:0/2701690723 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 msgr2=0x563591bdf190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 -- 172.21.15.40:0/2701690723 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x563591bce5e0 con 0x563591bd6be0 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16daffd700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009230 tx=0x7f16cc005dc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3ff024e45e4c5c6f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16c27fc700 1 -- 172.21.15.40:0/2701690723 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16cc00ba50 con 0x563591bd6be0 2023-07-14T11:13:08.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16c27fc700 1 -- 172.21.15.40:0/2701690723 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f16cc027040 con 0x563591bd6be0 2023-07-14T11:13:08.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.271+0000 7f16c27fc700 1 -- 172.21.15.40:0/2701690723 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16cc005400 con 0x563591bd6be0 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 -- 172.21.15.40:0/2701690723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 msgr2=0x563591be84a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 crypto rx=0x7f16cc009230 tx=0x7f16cc005dc0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 -- 172.21.15.40:0/2701690723 shutdown_connections 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591bda2e0 0x563591bdbb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bd6be0 0x563591be84a0 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2701690723 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591be1ad0 0x563591bdf190 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 -- 172.21.15.40:0/2701690723 >> 172.21.15.40:0/2701690723 conn(0x563591bccda0 msgr2=0x563591bd9bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 -- 172.21.15.40:0/2701690723 shutdown_connections 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.272+0000 7f16f5546b80 1 -- 172.21.15.40:0/2701690723 wait complete. 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.273+0000 7f16f5546b80 1 -- start start 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.273+0000 7f16f5546b80 1 Processor -- start 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd6be0 0x563591be6170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 0x563591c44120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591be6e30 con 0x563591bd6be0 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591c44690 con 0x563591be1ad0 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x563591c44850 con 0x563591bda2e0 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16f5546b80 10 client.0 _ll_register_callbacks cb 0x7f16cc00f770 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50716/0 (socket says 172.21.15.40:50716) 2023-07-14T11:13:08.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16daffd700 1 -- 172.21.15.40:0/2054994324 learned_addr learned my addr 172.21.15.40:0/2054994324 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.274+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd6be0 0x563591be6170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16dbfff700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 0x563591c44120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 -- 172.21.15.40:0/2054994324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 msgr2=0x563591c44120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 0x563591c44120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 -- 172.21.15.40:0/2054994324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd6be0 msgr2=0x563591be6170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd6be0 0x563591be6170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 -- 172.21.15.40:0/2054994324 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16dc1a9130 con 0x563591bda2e0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16dbfff700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 0x563591c44120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.275+0000 7f16daffd700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f16d00ca840 tx=0x7f16d0099d80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=39501dfc10815cee server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16baffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16d00cd2e0 con 0x563591bda2e0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f16d00a2600 con 0x563591bda2e0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f16d00cd440 con 0x563591bda2e0 2023-07-14T11:13:08.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f16d00996a0 con 0x563591bda2e0 2023-07-14T11:13:08.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16f5546b80 10 client.25954 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.276+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x56359193fc60 con 0x563591bda2e0 2023-07-14T11:13:08.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f16d00a2600 con 0x563591bda2e0 2023-07-14T11:13:08.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f16d00a2170 con 0x563591bda2e0 2023-07-14T11:13:08.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 10 client.25954 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 10 client.25954 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 20 client.25954 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591c45c30 0x563591c48040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 4 client.25954 mds_command: new command op to 24511 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x563591ae4050 con 0x563591c45c30 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.277+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591c45c30 0x563591c48040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.278+0000 7f16baffd700 10 client.25954 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.278+0000 7f16db7fe700 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591c45c30 0x563591c48040 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=386936e725831295 server_cookie=3292ca9cbc1d9f8f in_seq=0 out_seq=0 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.278+0000 7f16baffd700 1 -- 172.21.15.40:0/2054994324 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x563591ae4050 con 0x563591c45c30 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.278+0000 7f16baffd700 10 client.25954 handle_command_reply: tid=0 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.278+0000 7f16f5546b80 1 client.25954 shutdown 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 msgr2=0x563591be66b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 crypto rx=0x7f16d00ca840 tx=0x7f16d0099d80 comp rx=0 tx=0).stop 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 shutdown_connections 2023-07-14T11:13:08.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x563591c45c30 0x563591c48040 crc :-1 s=READY pgs=37 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x563591be1ad0 0x563591c44120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x563591bd6be0 0x563591be6170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 --2- 172.21.15.40:0/2054994324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x563591bda2e0 0x563591be66b0 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 >> 172.21.15.40:0/2054994324 conn(0x563591bccda0 msgr2=0x563591bdee40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16baffd700 0 client.25954 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 shutdown_connections 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 1 -- 172.21.15.40:0/2054994324 wait complete. 2023-07-14T11:13:08.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.279+0000 7f16f5546b80 20 client.25954 trim_cache size 0 max 16384 2023-07-14T11:13:08.288 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:13:08.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 msgr2=0x7f16c40a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 0x7f16c40a3910 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f16cc005e90 tx=0x7f16cc019040 comp rx=0 tx=0).stop 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 msgr2=0x7f16dc19c840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f16d000e9c0 tx=0x7f16d000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 shutdown_connections 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16c40a1460 0x7f16c40a3910 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16dc103e40 0x7f16dc1a0ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16dc0fea90 0x7f16dc19c300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 --2- 172.21.15.40:0/2987900526 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16dc0ff3a0 0x7f16dc19c840 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 >> 172.21.15.40:0/2987900526 conn(0x7f16dc0fa4b0 msgr2=0x7f16dc101990 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.280+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 shutdown_connections 2023-07-14T11:13:08.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.281+0000 7f16e1bff700 1 -- 172.21.15.40:0/2987900526 wait complete. 2023-07-14T11:13:08.306 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:3 damage ls 2023-07-14T11:13:08.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 -- 172.21.15.40:0/3511523324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c810a070 msgr2=0x7fe8c810c4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/3511523324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c810a070 0x7fe8c810c4b0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 crypto rx=0x7fe8b800b0a0 tx=0x7fe8b800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 -- 172.21.15.40:0/3511523324 shutdown_connections 2023-07-14T11:13:08.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/3511523324 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c810a070 0x7fe8c810c4b0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/3511523324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/3511523324 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 0x7fe8c8109aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.583+0000 7fe8ccbff700 1 -- 172.21.15.40:0/3511523324 >> 172.21.15.40:0/3511523324 conn(0x7fe8c80fa4b0 msgr2=0x7fe8c80fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.584+0000 7fe8ccbff700 1 -- 172.21.15.40:0/3511523324 shutdown_connections 2023-07-14T11:13:08.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.584+0000 7fe8ccbff700 1 -- 172.21.15.40:0/3511523324 wait complete. 2023-07-14T11:13:08.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 Processor -- start 2023-07-14T11:13:08.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 -- start start 2023-07-14T11:13:08.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 0x7fe8c8198440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 0x7fe8c819c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8c819cee0 con 0x7fe8c810a070 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8c819d050 con 0x7fe8c81015b0 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.585+0000 7fe8ccbff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8c819d210 con 0x7fe8c8100ca0 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50730/0 (socket says 172.21.15.40:50730) 2023-07-14T11:13:08.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 -- 172.21.15.40:0/2298539440 learned_addr learned my addr 172.21.15.40:0/2298539440 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 0x7fe8c8198440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 0x7fe8c819c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 -- 172.21.15.40:0/2298539440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 msgr2=0x7fe8c8198440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 0x7fe8c8198440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 -- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 msgr2=0x7fe8c819c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 0x7fe8c819c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c659c700 1 -- 172.21.15.40:0/2298539440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8b8009d00 con 0x7fe8c8100ca0 2023-07-14T11:13:08.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.586+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 0x7fe8c819c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:08.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8c659c700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fe8c000d950 tx=0x7fe8c000dc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8bf7fe700 1 -- 172.21.15.40:0/2298539440 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8c0009940 con 0x7fe8c8100ca0 2023-07-14T11:13:08.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8bf7fe700 1 -- 172.21.15.40:0/2298539440 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8c0010460 con 0x7fe8c8100ca0 2023-07-14T11:13:08.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8bf7fe700 1 -- 172.21.15.40:0/2298539440 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8c000f620 con 0x7fe8c8100ca0 2023-07-14T11:13:08.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8c819d4f0 con 0x7fe8c8100ca0 2023-07-14T11:13:08.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.587+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8c81a4d30 con 0x7fe8c8100ca0 2023-07-14T11:13:08.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.594+0000 7fe8bf7fe700 1 -- 172.21.15.40:0/2298539440 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7fe8c00108d0 con 0x7fe8c8100ca0 2023-07-14T11:13:08.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.595+0000 7fe8e04ffb80 1 Processor -- start 2023-07-14T11:13:08.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.595+0000 7fe8e04ffb80 1 -- start start 2023-07-14T11:13:08.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c5f6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c72da0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426b65cb0 con 0x55d426c5f2d0 2023-07-14T11:13:08.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426bb8a40 con 0x55d426c732e0 2023-07-14T11:13:08.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426bb1ed0 con 0x55d426c5fbe0 2023-07-14T11:13:08.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8c5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8c6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c5f6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8c5d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49372/0 (socket says 172.21.15.40:49372) 2023-07-14T11:13:08.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/141870046 learned_addr learned my addr 172.21.15.40:0/141870046 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.596+0000 7fe8c659c700 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c72da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.597+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/141870046 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 msgr2=0x55d426c72da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c72da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/141870046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 msgr2=0x55d426c5f6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c5f6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/141870046 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d4269dfde0 con 0x55d426c732e0 2023-07-14T11:13:08.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c5f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8bf7fe700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 0x7fe8b40a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fe8b000ac00 tx=0x7fe8b000af10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9e78efa23197da92 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8bf7fe700 1 -- 172.21.15.40:0/2298539440 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fe8c00cd100 con 0x7fe8c8100ca0 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.598+0000 7fe8bcff9700 1 -- 172.21.15.40:0/141870046 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b000fd60 con 0x55d426c732e0 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.599+0000 7fe8bcff9700 1 -- 172.21.15.40:0/141870046 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8b000fec0 con 0x55d426c732e0 2023-07-14T11:13:08.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.599+0000 7fe8bcff9700 1 -- 172.21.15.40:0/141870046 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b0010790 con 0x55d426c732e0 2023-07-14T11:13:08.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/141870046 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 msgr2=0x55d426c756c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fe8b000ac00 tx=0x7fe8b000af10 comp rx=0 tx=0).stop 2023-07-14T11:13:08.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 0x7fe8b40a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/141870046 shutdown_connections 2023-07-14T11:13:08.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c756c0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c5f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/141870046 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c72da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.600+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/141870046 >> 172.21.15.40:0/141870046 conn(0x55d426c55da0 msgr2=0x55d426c5b6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.601+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/141870046 shutdown_connections 2023-07-14T11:13:08.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.601+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/141870046 wait complete. 2023-07-14T11:13:08.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.601+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 0x7fe8b40a3910 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe8b8009cd0 tx=0x7fe8b8009b80 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.601+0000 7fe8e04ffb80 1 -- start start 2023-07-14T11:13:08.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 Processor -- start 2023-07-14T11:13:08.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c72cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c6ff50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426b51ac0 con 0x55d426c5f2d0 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426b517c0 con 0x55d426c732e0 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.602+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426a35920 con 0x55d426c5fbe0 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8e04ffb80 10 client.0 _ll_register_callbacks cb 0x55d426b4f760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8c6d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8c6d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49388/0 (socket says 172.21.15.40:49388) 2023-07-14T11:13:08.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/36950226 learned_addr learned my addr 172.21.15.40:0/36950226 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c6ff50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.603+0000 7fe8c659c700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c72cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/36950226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 msgr2=0x55d426c6ff50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c6ff50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/36950226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 msgr2=0x55d426c72cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c72cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/36950226 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8c80fe9e0 con 0x55d426c732e0 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c6ff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c659c700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c72cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fe8b8007780 tx=0x7fe8b80099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9af6f15c28e684bd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.604+0000 7fe8adffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b800e040 con 0x55d426c732e0 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fe8b8005550 con 0x55d426c732e0 2023-07-14T11:13:08.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d426baaeb0 con 0x55d426c732e0 2023-07-14T11:13:08.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe8b8005710 con 0x55d426c732e0 2023-07-14T11:13:08.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8e04ffb80 10 client.25973 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.605+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x7fe8b000a830 con 0x55d426c732e0 2023-07-14T11:13:08.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d426baaeb0 con 0x55d426c732e0 2023-07-14T11:13:08.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7fe8b802e020 con 0x55d426c732e0 2023-07-14T11:13:08.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 10 client.25973 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 10 client.25973 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 20 client.25973 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c82b50 0x55d426c84f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 4 client.25973 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.606+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d426b6d050 con 0x55d426c82b50 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.607+0000 7fe8c659c700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c82b50 0x55d426c84f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.607+0000 7fe8c659c700 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c82b50 0x55d426c84f60 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=46213dd0483e9ca8 server_cookie=45b6bccaa96f8024 in_seq=0 out_seq=0 2023-07-14T11:13:08.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.607+0000 7fe8adffb700 10 client.25973 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8adffb700 1 -- 172.21.15.40:0/36950226 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55d426b6d050 con 0x55d426c82b50 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8adffb700 10 client.25973 handle_command_reply: tid=0 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 client.25973 shutdown 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 msgr2=0x55d426c705d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fe8b8007780 tx=0x7fe8b80099e0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 shutdown_connections 2023-07-14T11:13:08.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c82b50 0x55d426c84f60 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c732e0 0x55d426c705d0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c5f2d0 0x55d426c72cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/36950226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5fbe0 0x55d426c6ff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 >> 172.21.15.40:0/36950226 conn(0x55d426c55da0 msgr2=0x55d426c5b230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.608+0000 7fe8adffb700 0 client.25973 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.609+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 shutdown_connections 2023-07-14T11:13:08.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.609+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/36950226 wait complete. 2023-07-14T11:13:08.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.609+0000 7fe8e04ffb80 20 client.25973 trim_cache size 0 max 16384 2023-07-14T11:13:08.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 Processor -- start 2023-07-14T11:13:08.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 -- start start 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426c79d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426c6d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426c70f30 con 0x55d426c55da0 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426c6e4b0 con 0x55d426c5fbe0 2023-07-14T11:13:08.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426c601c0 con 0x55d426c5f2d0 2023-07-14T11:13:08.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8c5d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8c5d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50768/0 (socket says 172.21.15.40:50768) 2023-07-14T11:13:08.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.631+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/3716783878 learned_addr learned my addr 172.21.15.40:0/3716783878 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426c6d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c659c700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426c79d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/3716783878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 msgr2=0x55d426c6d1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426c6d1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/3716783878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 msgr2=0x55d426c79d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426c79d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 -- 172.21.15.40:0/3716783878 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d4269c8c60 con 0x55d426c5f2d0 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426c6d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c659c700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426c79d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x55d426c732b0 tx=0x7fe8b0018ee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=41232000cd88cb0e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.632+0000 7fe8bd7fa700 1 -- 172.21.15.40:0/3716783878 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b0005240 con 0x55d426c5f2d0 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.633+0000 7fe8bd7fa700 1 -- 172.21.15.40:0/3716783878 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fe8b0005d70 con 0x55d426c5f2d0 2023-07-14T11:13:08.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.633+0000 7fe8bd7fa700 1 -- 172.21.15.40:0/3716783878 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b00186a0 con 0x55d426c5f2d0 2023-07-14T11:13:08.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.633+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3716783878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 msgr2=0x55d426c72870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.633+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x55d426c732b0 tx=0x7fe8b0018ee0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3716783878 shutdown_connections 2023-07-14T11:13:08.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426c6d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426c79d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3716783878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426c72870 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3716783878 >> 172.21.15.40:0/3716783878 conn(0x55d426c57600 msgr2=0x55d426c6b260 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.634+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3716783878 shutdown_connections 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.635+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3716783878 wait complete. 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.635+0000 7fe8e04ffb80 1 -- start start 2023-07-14T11:13:08.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.635+0000 7fe8e04ffb80 1 Processor -- start 2023-07-14T11:13:08.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426ccabd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426ccb110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426ccf260 con 0x55d426c55da0 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426ccf3d0 con 0x55d426c5fbe0 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d426ccf590 con 0x55d426c5f2d0 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8e04ffb80 10 client.0 _ll_register_callbacks cb 0x7fe8b8004e80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8c6d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.636+0000 7fe8c6d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49404/0 (socket says 172.21.15.40:49404) 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/3003640510 learned_addr learned my addr 172.21.15.40:0/3003640510 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426ccb110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/3003640510 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 msgr2=0x55d426ccb110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426ccb110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/3003640510 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 msgr2=0x55d426ccabd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426ccabd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 -- 172.21.15.40:0/3003640510 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d4269dfde0 con 0x55d426c5fbe0 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c5d9b700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426ccb110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.641 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8c6d9d700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fe8b800b8b0 tx=0x7fe8b8029d10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ca643fdc4dab69af server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.637+0000 7fe8adffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:08.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fe8b800e070 con 0x55d426c5fbe0 2023-07-14T11:13:08.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fe8b8005560 con 0x55d426c5fbe0 2023-07-14T11:13:08.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d426b0b4f0 con 0x55d426c5fbe0 2023-07-14T11:13:08.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7fe8b8005720 con 0x55d426c5fbe0 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 10 client.25976 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55d426a06250 con 0x55d426c5fbe0 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d426b0b4f0 con 0x55d426c5fbe0 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7fe8b802e070 con 0x55d426c5fbe0 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 10 client.25976 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 10 client.25976 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 20 client.25976 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c732e0 0x55d426ceab30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.638+0000 7fe8e04ffb80 4 client.25976 mds_command: new command op to 24521 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.639+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d426a00f70 con 0x55d426c732e0 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.639+0000 7fe8c659c700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c732e0 0x55d426ceab30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8c659c700 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c732e0 0x55d426ceab30 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=add20f94bcb43de9 server_cookie=f99c06507e58d6f1 in_seq=0 out_seq=0 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8adffb700 10 client.25976 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:13:08.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8adffb700 1 -- 172.21.15.40:0/3003640510 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d426a00f70 con 0x55d426c732e0 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8adffb700 10 client.25976 handle_command_reply: tid=0 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8e04ffb80 1 client.25976 shutdown 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.640+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 msgr2=0x55d426ccecb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7fe8b800b8b0 tx=0x7fe8b8029d10 comp rx=0 tx=0).stop 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 shutdown_connections 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x55d426c732e0 0x55d426ceab30 crc :-1 s=READY pgs=43 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d426c5fbe0 0x55d426ccecb0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d426c55da0 0x55d426ccabd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 --2- 172.21.15.40:0/3003640510 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d426c5f2d0 0x55d426ccb110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8adffb700 0 client.25976 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 >> 172.21.15.40:0/3003640510 conn(0x55d426c57600 msgr2=0x55d426c612e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 shutdown_connections 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 1 -- 172.21.15.40:0/3003640510 wait complete. 2023-07-14T11:13:08.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.641+0000 7fe8e04ffb80 20 client.25976 trim_cache size 0 max 16384 2023-07-14T11:13:08.648 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:13:08.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 msgr2=0x7fe8b40a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 0x7fe8b40a3910 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fe8b8009cd0 tx=0x7fe8b8009b80 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 msgr2=0x7fe8c8197f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 crypto rx=0x7fe8c000d950 tx=0x7fe8c000dc60 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 shutdown_connections 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fe8b40a1460 0x7fe8b40a3910 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8c810a070 0x7fe8c819c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fe8c8100ca0 0x7fe8c8197f00 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 --2- 172.21.15.40:0/2298539440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fe8c81015b0 0x7fe8c8198440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.642+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 >> 172.21.15.40:0/2298539440 conn(0x7fe8c80fa4b0 msgr2=0x7fe8c8101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.643+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 shutdown_connections 2023-07-14T11:13:08.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.643+0000 7fe8ccbff700 1 -- 172.21.15.40:0/2298539440 wait complete. 2023-07-14T11:13:08.669 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2023-07-14T11:13:08.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 -- 172.21.15.40:0/2178157433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 msgr2=0x7f7e100fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 --2- 172.21.15.40:0/2178157433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e100fff50 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc0098f0 tx=0x7f7dfc009c00 comp rx=0 tx=0).stop 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 -- 172.21.15.40:0/2178157433 shutdown_connections 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 --2- 172.21.15.40:0/2178157433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1010c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 --2- 172.21.15.40:0/2178157433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 0x7f7e10105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 --2- 172.21.15.40:0/2178157433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e100fff50 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 -- 172.21.15.40:0/2178157433 >> 172.21.15.40:0/2178157433 conn(0x7f7e100fb5e0 msgr2=0x7f7e100fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 -- 172.21.15.40:0/2178157433 shutdown_connections 2023-07-14T11:13:08.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.939+0000 7f7e171ff700 1 -- 172.21.15.40:0/2178157433 wait complete. 2023-07-14T11:13:08.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.940+0000 7f7e171ff700 1 Processor -- start 2023-07-14T11:13:08.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 -- start start 2023-07-14T11:13:08.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e1019ba40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1019bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 0x7f7e10195ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e10196120 con 0x7f7e10105af0 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e10196290 con 0x7f7e10100490 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.941+0000 7f7e171ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e10196450 con 0x7f7e100ffae0 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e14f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e1019ba40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e14f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e1019ba40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50794/0 (socket says 172.21.15.40:50794) 2023-07-14T11:13:08.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e14f9b700 1 -- 172.21.15.40:0/1071210836 learned_addr learned my addr 172.21.15.40:0/1071210836 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1019bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e1579c700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 0x7f7e10195ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1071210836 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 msgr2=0x7f7e1019ba40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e1019ba40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 msgr2=0x7f7e10195ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 0x7f7e10195ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.942+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1071210836 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e08009d00 con 0x7f7e10100490 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1019bf80 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc009810 tx=0x7f7dfc00fa00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e0dffb700 1 -- 172.21.15.40:0/1071210836 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc01c070 con 0x7f7e10100490 2023-07-14T11:13:08.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7dfc009520 con 0x7f7e10100490 2023-07-14T11:13:08.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e0dffb700 1 -- 172.21.15.40:0/1071210836 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dfc00bba0 con 0x7f7e10100490 2023-07-14T11:13:08.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e0dffb700 1 -- 172.21.15.40:0/1071210836 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc003970 con 0x7f7e10100490 2023-07-14T11:13:08.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.943+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e101a2850 con 0x7f7e10100490 2023-07-14T11:13:08.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.948+0000 7f7e0dffb700 1 -- 172.21.15.40:0/1071210836 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f7dfc020030 con 0x7f7e10100490 2023-07-14T11:13:08.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.948+0000 7f7e2abb5b80 1 Processor -- start 2023-07-14T11:13:08.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 -- start start 2023-07-14T11:13:08.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 0x55c9bd45b6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd361cb0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd3b4a40 con 0x55c9bd45b2d0 2023-07-14T11:13:08.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.949+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd3aded0 con 0x55c9bd466ad0 2023-07-14T11:13:08.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50800/0 (socket says 172.21.15.40:50800) 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2869978536 learned_addr learned my addr 172.21.15.40:0/2869978536 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e1579c700 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 0x55c9bd45b6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2869978536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 msgr2=0x55c9bd45b6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 0x55c9bd45b6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2869978536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 msgr2=0x55c9bd466590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2869978536 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c9bd1dbde0 con 0x55c9bd466ad0 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e1579c700 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 0x55c9bd45b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.950+0000 7f7e0dffb700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 0x7f7e000a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.951+0000 7f7e0dffb700 1 -- 172.21.15.40:0/1071210836 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7dfc0ce060 con 0x7f7e10100490 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.951+0000 7f7e1579c700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 0x7f7e000a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.951+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc0cb260 tx=0x7f7dfc0d1f70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=1eff185dff35869e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.951+0000 7f7dfaffd700 1 -- 172.21.15.40:0/2869978536 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc096f40 con 0x55c9bd466ad0 2023-07-14T11:13:08.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.952+0000 7f7dfaffd700 1 -- 172.21.15.40:0/2869978536 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dfc09ee90 con 0x55c9bd466ad0 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.952+0000 7f7dfaffd700 1 -- 172.21.15.40:0/2869978536 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc09ac00 con 0x55c9bd466ad0 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.952+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2869978536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 msgr2=0x55c9bd468eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.952+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc0cb260 tx=0x7f7dfc0d1f70 comp rx=0 tx=0).stop 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.952+0000 7f7e1579c700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 0x7f7e000a3910 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f7e1019c740 tx=0x7f7e0800b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2869978536 shutdown_connections 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd466ad0 0x55c9bd468eb0 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45b2d0 0x55c9bd45b6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2869978536 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2869978536 >> 172.21.15.40:0/2869978536 conn(0x55c9bd451da0 msgr2=0x55c9bd4576a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2869978536 shutdown_connections 2023-07-14T11:13:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.953+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2869978536 wait complete. 2023-07-14T11:13:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.954+0000 7f7e2abb5b80 1 -- start start 2023-07-14T11:13:08.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.954+0000 7f7e2abb5b80 1 Processor -- start 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd465fc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 0x55c9bd45f400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd34dac0 con 0x55c9bd466ad0 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd34d7c0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd231920 con 0x55c9bd45b2d0 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e2abb5b80 10 client.0 _ll_register_callbacks cb 0x55c9bd34b760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49460/0 (socket says 172.21.15.40:49460) 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1284625850 learned_addr learned my addr 172.21.15.40:0/1284625850 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.955+0000 7f7e1579c700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 0x55c9bd45f400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd465fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 msgr2=0x55c9bd465fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd465fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1284625850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 msgr2=0x55c9bd45f400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 0x55c9bd45f400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 -- 172.21.15.40:0/1284625850 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e101a2850 con 0x55c9bd45bbe0 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e1579c700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 0x55c9bd45f400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd465fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc0092a0 tx=0x7f7dfc00fa70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e10bf911adedac89 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7deffff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc09a740 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7dfc09ec30 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.956+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c9bd3074f0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7dfc096f40 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 10 client.25988 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55c9bd1c4c60 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c9bd3074f0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f7dfc097310 con 0x55c9bd45bbe0 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 10 client.25988 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 10 client.25988 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 20 client.25988 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd47e480 0x55c9bd480860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 4 client.25988 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:08.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.957+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c9bd3a6eb0 con 0x55c9bd47e480 2023-07-14T11:13:08.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.958+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd47e480 0x55c9bd480860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.958+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd47e480 0x55c9bd480860 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=4c309ffce7ee0ae2 server_cookie=3366957cda26e528 in_seq=0 out_seq=0 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.958+0000 7f7deffff700 10 client.25988 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.959+0000 7f7deffff700 1 -- 172.21.15.40:0/1284625850 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c9bd3a6eb0 con 0x55c9bd47e480 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.959+0000 7f7deffff700 10 client.25988 handle_command_reply: tid=0 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.959+0000 7f7e2abb5b80 1 client.25988 shutdown 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 msgr2=0x55c9bd466500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc0092a0 tx=0x7f7dfc00fa70 comp rx=0 tx=0).stop 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 shutdown_connections 2023-07-14T11:13:08.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd47e480 0x55c9bd480860 crc :-1 s=READY pgs=38 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd466ad0 0x55c9bd45f400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd465fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/1284625850 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd45bbe0 0x55c9bd466500 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 >> 172.21.15.40:0/1284625850 conn(0x55c9bd451da0 msgr2=0x55c9bd457290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7deffff700 0 client.25988 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 shutdown_connections 2023-07-14T11:13:08.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/1284625850 wait complete. 2023-07-14T11:13:08.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.960+0000 7f7e2abb5b80 20 client.25988 trim_cache size 0 max 16384 2023-07-14T11:13:08.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 Processor -- start 2023-07-14T11:13:08.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 -- start start 2023-07-14T11:13:08.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45b2d0 0x55c9bd468cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd46d1c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd4519b0 con 0x55c9bd45b2d0 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd4584d0 con 0x55c9bd466ad0 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.979+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd45fa00 con 0x55c9bd45bbe0 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e14f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45b2d0 0x55c9bd468cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50818/0 (socket says 172.21.15.40:50818) 2023-07-14T11:13:08.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2182178960 learned_addr learned my addr 172.21.15.40:0/2182178960 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e1579c700 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd46d1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2182178960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 msgr2=0x55c9bd46d1c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd46d1c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2182178960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45b2d0 msgr2=0x55c9bd468cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45b2d0 0x55c9bd468cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 -- 172.21.15.40:0/2182178960 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c9bd202250 con 0x55c9bd45bbe0 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.980+0000 7f7e0ffff700 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc097670 tx=0x7f7dfc0991b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4f48dbaab37916b5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.981+0000 7f7dfb7fe700 1 -- 172.21.15.40:0/2182178960 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc0d1ea0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.981+0000 7f7dfb7fe700 1 -- 172.21.15.40:0/2182178960 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dfc096f40 con 0x55c9bd45bbe0 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.981+0000 7f7dfb7fe700 1 -- 172.21.15.40:0/2182178960 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7dfc0a2fc0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2182178960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 msgr2=0x55c9bd46cc80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc097670 tx=0x7f7dfc0991b0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2182178960 shutdown_connections 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd46d1c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45b2d0 0x55c9bd468cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/2182178960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45bbe0 0x55c9bd46cc80 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2182178960 >> 172.21.15.40:0/2182178960 conn(0x55c9bd451da0 msgr2=0x55c9bd455e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2182178960 shutdown_connections 2023-07-14T11:13:08.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.982+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/2182178960 wait complete. 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.983+0000 7f7e2abb5b80 1 -- start start 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.983+0000 7f7e2abb5b80 1 Processor -- start 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd47b050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd47b590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd46a8f0 con 0x55c9bd45bbe0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd46aa60 con 0x55c9bd466ad0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c9bd46ac20 con 0x55c9bd45b2d0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e2abb5b80 10 client.0 _ll_register_callbacks cb 0x55c9bd34b760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e1579c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd47b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e1579c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49478/0 (socket says 172.21.15.40:49478) 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e0ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd47b590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45114/0 (socket says 172.21.15.40:45114) 2023-07-14T11:13:08.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.984+0000 7f7e1579c700 1 -- 172.21.15.40:0/706070851 learned_addr learned my addr 172.21.15.40:0/706070851 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 -- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 msgr2=0x55c9bd47b050 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd47b050 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 -- 172.21.15.40:0/706070851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 msgr2=0x55c9bd47b590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd47b590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 -- 172.21.15.40:0/706070851 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c9bd1c4c60 con 0x55c9bd466ad0 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7e1579c700 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f7e08016b90 tx=0x7f7e08004b40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3ef891c7e6743fb4 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:08.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.985+0000 7f7deffff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:08.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7e0800e040 con 0x55c9bd466ad0 2023-07-14T11:13:08.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7e08003c00 con 0x55c9bd466ad0 2023-07-14T11:13:08.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7dfc09ee90 con 0x55c9bd466ad0 2023-07-14T11:13:08.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7e08003dc0 con 0x55c9bd466ad0 2023-07-14T11:13:08.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7e2abb5b80 10 client.25994 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.986+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55c9bd1da2b0 con 0x55c9bd466ad0 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.987+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7dfc09ee90 con 0x55c9bd466ad0 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f7e0802b030 con 0x55c9bd466ad0 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 10 client.25994 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 10 client.25994 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 20 client.25994 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:08.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd46c330 0x55c9bd4d52a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 4 client.25994 mds_command: new command op to 24523 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55c9bd369050 con 0x55c9bd46c330 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.988+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd46c330 0x55c9bd4d52a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e14f9b700 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd46c330 0x55c9bd4d52a0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=8ab2b66e0b055f9c server_cookie=5f75725a3178c62d in_seq=0 out_seq=0 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7deffff700 10 client.25994 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7deffff700 1 -- 172.21.15.40:0/706070851 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55c9bd369050 con 0x55c9bd46c330 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7deffff700 10 client.25994 handle_command_reply: tid=0 2023-07-14T11:13:08.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 client.25994 shutdown 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 msgr2=0x55c9bd47bad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f7e08016b90 tx=0x7f7e08004b40 comp rx=0 tx=0).stop 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 shutdown_connections 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55c9bd46c330 0x55c9bd4d52a0 crc :-1 s=READY pgs=39 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c9bd466ad0 0x55c9bd47bad0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c9bd45b2d0 0x55c9bd47b050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 --2- 172.21.15.40:0/706070851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c9bd45bbe0 0x55c9bd47b590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 >> 172.21.15.40:0/706070851 conn(0x55c9bd451da0 msgr2=0x55c9bd46cc80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7deffff700 0 client.25994 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:13:08.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.989+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 shutdown_connections 2023-07-14T11:13:08.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.990+0000 7f7e2abb5b80 1 -- 172.21.15.40:0/706070851 wait complete. 2023-07-14T11:13:08.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.990+0000 7f7e2abb5b80 20 client.25994 trim_cache size 0 max 16384 2023-07-14T11:13:08.997 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:13:08.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 msgr2=0x7f7e000a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 0x7f7e000a3910 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f7e1019c740 tx=0x7f7e0800b040 comp rx=0 tx=0).stop 2023-07-14T11:13:08.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 msgr2=0x7f7e1019bf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1019bf80 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f7dfc009810 tx=0x7f7dfc00fa00 comp rx=0 tx=0).stop 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 shutdown_connections 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7e000a1460 0x7f7e000a3910 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e10105af0 0x7f7e10195ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7e100ffae0 0x7f7e1019ba40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 --2- 172.21.15.40:0/1071210836 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7e10100490 0x7f7e1019bf80 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:08.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 >> 172.21.15.40:0/1071210836 conn(0x7f7e100fb5e0 msgr2=0x7f7e101035f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:08.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 shutdown_connections 2023-07-14T11:13:08.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:08.991+0000 7f7e171ff700 1 -- 172.21.15.40:0/1071210836 wait complete. 2023-07-14T11:13:09.014 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:08 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/916549396' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:13:09.017 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2023-07-14T11:13:09.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:08 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/916549396' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:13:09.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:08 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/916549396' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:13:09.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.269+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/3582211143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40728b0 msgr2=0x7f4dd4072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.269+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/3582211143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40728b0 0x7f4dd4072c80 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8009990 tx=0x7f4dc8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/3582211143 shutdown_connections 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/3582211143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd4110ae0 0x7f4dd4112ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/3582211143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40728b0 0x7f4dd4072c80 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/3582211143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd40731c0 0x7f4dd41105a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/3582211143 >> 172.21.15.40:0/3582211143 conn(0x7f4dd40fb5f0 msgr2=0x7f4dd40fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/3582211143 shutdown_connections 2023-07-14T11:13:09.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.270+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/3582211143 wait complete. 2023-07-14T11:13:09.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.271+0000 7f4ddb1ff700 1 Processor -- start 2023-07-14T11:13:09.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 -- start start 2023-07-14T11:13:09.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 0x7f4dd419c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd4110ae0 0x7f4dd41a0c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd41a1290 con 0x7f4dd4110ae0 2023-07-14T11:13:09.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd41a1400 con 0x7f4dd40728b0 2023-07-14T11:13:09.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4ddb1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4dd41a15c0 con 0x7f4dd40731c0 2023-07-14T11:13:09.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4dd3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4dd3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50834/0 (socket says 172.21.15.40:50834) 2023-07-14T11:13:09.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.272+0000 7f4dd3fff700 1 -- 172.21.15.40:0/1262539988 learned_addr learned my addr 172.21.15.40:0/1262539988 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 0x7f4dd419c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd979c700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd4110ae0 0x7f4dd41a0c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 -- 172.21.15.40:0/1262539988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 msgr2=0x7f4dd419c2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 0x7f4dd419c2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 -- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd4110ae0 msgr2=0x7f4dd41a0c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd4110ae0 0x7f4dd41a0c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 -- 172.21.15.40:0/1262539988 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4dc80095f0 con 0x7f4dd40731c0 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 0x7f4dd419c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f4dc400e9a0 tx=0x7f4dc400ed60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd1ffb700 1 -- 172.21.15.40:0/1262539988 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc400cbd0 con 0x7f4dd40731c0 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd1ffb700 1 -- 172.21.15.40:0/1262539988 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dc400cd30 con 0x7f4dd40731c0 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4dd41a18a0 con 0x7f4dd40731c0 2023-07-14T11:13:09.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.273+0000 7f4dd1ffb700 1 -- 172.21.15.40:0/1262539988 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc4018680 con 0x7f4dd40731c0 2023-07-14T11:13:09.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.274+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4dd41a90e0 con 0x7f4dd40731c0 2023-07-14T11:13:09.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.279+0000 7f4dd1ffb700 1 -- 172.21.15.40:0/1262539988 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f4dc40187e0 con 0x7f4dd40731c0 2023-07-14T11:13:09.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.280+0000 7f4deebaab80 1 Processor -- start 2023-07-14T11:13:09.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.280+0000 7f4deebaab80 1 -- start start 2023-07-14T11:13:09.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa42590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa44eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f93dcb0 con 0x55d05fa37be0 2023-07-14T11:13:09.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f990a40 con 0x55d05fa372d0 2023-07-14T11:13:09.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4deebaab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f989ed0 con 0x55d05fa42ad0 2023-07-14T11:13:09.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4dd3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa44eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49498/0 (socket says 172.21.15.40:49498) 2023-07-14T11:13:09.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4dd979c700 1 -- 172.21.15.40:0/2132368498 learned_addr learned my addr 172.21.15.40:0/2132368498 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.281+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa42590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 -- 172.21.15.40:0/2132368498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 msgr2=0x55d05fa44eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa44eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 -- 172.21.15.40:0/2132368498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 msgr2=0x55d05fa42590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa42590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd1ffb700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 0x7f4dbc0a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 -- 172.21.15.40:0/2132368498 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d05f7b7de0 con 0x55d05fa372d0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa44eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd1ffb700 1 -- 172.21.15.40:0/1262539988 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4dc4014070 con 0x7f4dd40731c0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc0093e0 tx=0x7f4dcc009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=29793aa0860bc1bc server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.282+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 0x7f4dbc0a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.283+0000 7f4dbaffd700 1 -- 172.21.15.40:0/2132368498 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dcc00e070 con 0x55d05fa372d0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.283+0000 7f4dbaffd700 1 -- 172.21.15.40:0/2132368498 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dcc00f040 con 0x55d05fa372d0 2023-07-14T11:13:09.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.283+0000 7f4dbaffd700 1 -- 172.21.15.40:0/2132368498 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dcc0146d0 con 0x55d05fa372d0 2023-07-14T11:13:09.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 -- 172.21.15.40:0/2132368498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 msgr2=0x55d05fa376a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc0093e0 tx=0x7f4dcc009920 comp rx=0 tx=0).stop 2023-07-14T11:13:09.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4dd3fff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 0x7f4dbc0a3910 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f4dc40ca8e0 tx=0x7f4dc4009620 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 -- 172.21.15.40:0/2132368498 shutdown_connections 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa44eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa376a0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2132368498 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa42590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 -- 172.21.15.40:0/2132368498 >> 172.21.15.40:0/2132368498 conn(0x55d05fa2dda0 msgr2=0x55d05fa336a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 -- 172.21.15.40:0/2132368498 shutdown_connections 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.284+0000 7f4deebaab80 1 -- 172.21.15.40:0/2132368498 wait complete. 2023-07-14T11:13:09.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.285+0000 7f4deebaab80 1 -- start start 2023-07-14T11:13:09.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.285+0000 7f4deebaab80 1 Processor -- start 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3f730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa3fd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f929ac0 con 0x55d05fa37be0 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f9297c0 con 0x55d05fa372d0 2023-07-14T11:13:09.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05f80d920 con 0x55d05fa42ad0 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4deebaab80 10 client.0 _ll_register_callbacks cb 0x55d05f927760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49506/0 (socket says 172.21.15.40:49506) 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.286+0000 7f4dd979c700 1 -- 172.21.15.40:0/984875570 learned_addr learned my addr 172.21.15.40:0/984875570 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd979c700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa3fd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3f730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/984875570 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 msgr2=0x55d05fa3fd90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa3fd90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/984875570 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 msgr2=0x55d05fa424b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.287+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/984875570 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4dcc009090 con 0x55d05fa37be0 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4dd979c700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3f730 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8005b40 tx=0x7f4dc800bb30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=4c923c1a7708444d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4db3fff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:13:09.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc801e050 con 0x55d05fa37be0 2023-07-14T11:13:09.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4dc800f870 con 0x55d05fa37be0 2023-07-14T11:13:09.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.288+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d05f8e34f0 con 0x55d05fa37be0 2023-07-14T11:13:09.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4dc800fa30 con 0x55d05fa37be0 2023-07-14T11:13:09.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 10 client.16005 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:09.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55d05f7a0c60 con 0x55d05fa37be0 2023-07-14T11:13:09.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d05f8e34f0 con 0x55d05fa37be0 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f4dc8023c60 con 0x55d05fa37be0 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 10 client.16005 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 10 client.16005 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 20 client.16005 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05fa52330 0x55d05fa54740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 4 client.16005 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:09.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.289+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d05f982eb0 con 0x55d05fa52330 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.290+0000 7f4dd979c700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05fa52330 0x55d05fa54740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.291+0000 7f4db3fff700 10 client.16005 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.291+0000 7f4dd979c700 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05fa52330 0x55d05fa54740 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=b3f0242c65f00020 server_cookie=b67d5ff8685cafb6 in_seq=0 out_seq=0 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4db3fff700 1 -- 172.21.15.40:0/984875570 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55d05f982eb0 con 0x55d05fa52330 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4db3fff700 10 client.16005 handle_command_reply: tid=0 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 client.16005 shutdown 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 msgr2=0x55d05fa3f730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3f730 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8005b40 tx=0x7f4dc800bb30 comp rx=0 tx=0).stop 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 shutdown_connections 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05fa52330 0x55d05fa54740 crc :-1 s=READY pgs=40 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05fa3fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa424b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 --2- 172.21.15.40:0/984875570 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3f730 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 >> 172.21.15.40:0/984875570 conn(0x55d05fa2dda0 msgr2=0x55d05fa33230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.292+0000 7f4db3fff700 0 client.16005 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:13:09.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.293+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 shutdown_connections 2023-07-14T11:13:09.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.293+0000 7f4deebaab80 1 -- 172.21.15.40:0/984875570 wait complete. 2023-07-14T11:13:09.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.293+0000 7f4deebaab80 20 client.16005 trim_cache size 0 max 16384 2023-07-14T11:13:09.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.309+0000 7f4deebaab80 1 Processor -- start 2023-07-14T11:13:09.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 -- start start 2023-07-14T11:13:09.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa372d0 0x55d05fa42050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 0x55d05fa3f430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05fa40d10 con 0x55d05fa37be0 2023-07-14T11:13:09.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05fa52110 con 0x55d05fa42ad0 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4deebaab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05fa381c0 con 0x55d05fa372d0 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4dd8f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa372d0 0x55d05fa42050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4dd8f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45160/0 (socket says 172.21.15.40:45160) 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.310+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/1794896436 learned_addr learned my addr 172.21.15.40:0/1794896436 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd979c700 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 0x55d05fa3f430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/1794896436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa372d0 msgr2=0x55d05fa42050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa372d0 0x55d05fa42050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/1794896436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 msgr2=0x55d05fa3f430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 0x55d05fa3f430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 -- 172.21.15.40:0/1794896436 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d05f7de250 con 0x55d05fa37be0 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd979c700 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 0x55d05fa3f430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8017560 tx=0x7f4dc8009290 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f0de521e644d2186 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dbb7fe700 1 -- 172.21.15.40:0/1794896436 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc801e070 con 0x55d05fa37be0 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.311+0000 7f4dbb7fe700 1 -- 172.21.15.40:0/1794896436 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dc801f040 con 0x55d05fa37be0 2023-07-14T11:13:09.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.312+0000 7f4dbb7fe700 1 -- 172.21.15.40:0/1794896436 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dc8017aa0 con 0x55d05fa37be0 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 -- 172.21.15.40:0/1794896436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 msgr2=0x55d05fa3eef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 crypto rx=0x7f4dc8017560 tx=0x7f4dc8009290 comp rx=0 tx=0).stop 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 -- 172.21.15.40:0/1794896436 shutdown_connections 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa42ad0 0x55d05fa3f430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa372d0 0x55d05fa42050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 --2- 172.21.15.40:0/1794896436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa3eef0 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 -- 172.21.15.40:0/1794896436 >> 172.21.15.40:0/1794896436 conn(0x55d05fa2dda0 msgr2=0x55d05fa3aa30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 -- 172.21.15.40:0/1794896436 shutdown_connections 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.313+0000 7f4deebaab80 1 -- 172.21.15.40:0/1794896436 wait complete. 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.314+0000 7f4deebaab80 1 -- start start 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.314+0000 7f4deebaab80 1 Processor -- start 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.314+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.314+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa47c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4deebaab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05faa51f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4deebaab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05faa5730 con 0x55d05fa37be0 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4deebaab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05faa5870 con 0x55d05fa372d0 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4deebaab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d05faa59b0 con 0x55d05fa42ad0 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4deebaab80 10 client.0 _ll_register_callbacks cb 0x7f4dc8009810 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49530/0 (socket says 172.21.15.40:49530) 2023-07-14T11:13:09.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 -- 172.21.15.40:0/2688697480 learned_addr learned my addr 172.21.15.40:0/2688697480 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa47c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05faa51f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 -- 172.21.15.40:0/2688697480 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 msgr2=0x55d05faa51f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05faa51f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 -- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 msgr2=0x55d05fa47c00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa47c00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.315+0000 7f4dd979c700 1 -- 172.21.15.40:0/2688697480 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d05f7a0c60 con 0x55d05fa372d0 2023-07-14T11:13:09.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4dd979c700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc007ac0 tx=0x7f4dc8005a90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=95a3f926a9e75f50 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4db3fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:09.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4dcc00e040 con 0x55d05fa372d0 2023-07-14T11:13:09.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f4dcc00f040 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4dcc015500 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4dcc0156c0 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4deebaab80 10 client.26009 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.316+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55d05f7b62b0 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4dcc00f040 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f4dcc01f450 con 0x55d05fa372d0 2023-07-14T11:13:09.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 10 client.26009 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 10 client.26009 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 20 client.26009 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05faa7070 0x55d05faa9480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 4 client.26009 mds_command: new command op to 24565 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55d05f945050 con 0x55d05faa7070 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.317+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05faa7070 0x55d05faa9480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.318+0000 7f4dd8f9b700 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05faa7070 0x55d05faa9480 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=11385c36517ade92 server_cookie=442768b1ddf86450 in_seq=0 out_seq=0 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.318+0000 7f4db3fff700 10 client.26009 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.318+0000 7f4db3fff700 1 -- 172.21.15.40:0/2688697480 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55d05f945050 con 0x55d05faa7070 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.318+0000 7f4db3fff700 10 client.26009 handle_command_reply: tid=0 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.318+0000 7f4deebaab80 1 client.26009 shutdown 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 msgr2=0x55d05fa476c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f4dcc007ac0 tx=0x7f4dc8005a90 comp rx=0 tx=0).stop 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 shutdown_connections 2023-07-14T11:13:09.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x55d05faa7070 0x55d05faa9480 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d05fa42ad0 0x55d05faa51f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d05fa372d0 0x55d05fa476c0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 --2- 172.21.15.40:0/2688697480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d05fa37be0 0x55d05fa47c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4db3fff700 0 client.26009 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 >> 172.21.15.40:0/2688697480 conn(0x55d05fa2dda0 msgr2=0x55d05fa39270 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 shutdown_connections 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.319+0000 7f4deebaab80 1 -- 172.21.15.40:0/2688697480 wait complete. 2023-07-14T11:13:09.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.320+0000 7f4deebaab80 20 client.26009 trim_cache size 0 max 16384 2023-07-14T11:13:09.328 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:13:09.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.320+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 msgr2=0x7f4dbc0a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.320+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 0x7f4dbc0a3910 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f4dc40ca8e0 tx=0x7f4dc4009620 comp rx=0 tx=0).stop 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.320+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 msgr2=0x7f4dd419c7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.320+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f4dc400e9a0 tx=0x7f4dc400ed60 comp rx=0 tx=0).stop 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 shutdown_connections 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4dbc0a1460 0x7f4dbc0a3910 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4dd4110ae0 0x7f4dd41a0c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4dd40728b0 0x7f4dd419c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 --2- 172.21.15.40:0/1262539988 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4dd40731c0 0x7f4dd419c7f0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 >> 172.21.15.40:0/1262539988 conn(0x7f4dd40fb5f0 msgr2=0x7f4dd40fc250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 shutdown_connections 2023-07-14T11:13:09.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.321+0000 7f4ddb1ff700 1 -- 172.21.15.40:0/1262539988 wait complete. 2023-07-14T11:13:09.348 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2023-07-14T11:13:09.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 -- 172.21.15.40:0/3457911736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 msgr2=0x7f44c010b810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 --2- 172.21.15.40:0/3457911736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c010b810 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f44b800b0a0 tx=0x7f44b800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 -- 172.21.15.40:0/3457911736 shutdown_connections 2023-07-14T11:13:09.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 --2- 172.21.15.40:0/3457911736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c010b810 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 --2- 172.21.15.40:0/3457911736 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c01049e0 0x7f44c0104db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 --2- 172.21.15.40:0/3457911736 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c00fe9d0 0x7f44c00fee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 -- 172.21.15.40:0/3457911736 >> 172.21.15.40:0/3457911736 conn(0x7f44c00fa4b0 msgr2=0x7f44c00fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 -- 172.21.15.40:0/3457911736 shutdown_connections 2023-07-14T11:13:09.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.611+0000 7f44c87ff700 1 -- 172.21.15.40:0/3457911736 wait complete. 2023-07-14T11:13:09.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.612+0000 7f44c87ff700 1 Processor -- start 2023-07-14T11:13:09.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.612+0000 7f44c87ff700 1 -- start start 2023-07-14T11:13:09.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c00fe9d0 0x7f44c0197e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c01983b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44c019ce50 con 0x7f44c01049e0 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44c019cfc0 con 0x7f44c00ff380 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c87ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f44c019d180 con 0x7f44c00fe9d0 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c6d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c6d9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45194/0 (socket says 172.21.15.40:45194) 2023-07-14T11:13:09.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c6d9c700 1 -- 172.21.15.40:0/2438361839 learned_addr learned my addr 172.21.15.40:0/2438361839 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c659b700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c00fe9d0 0x7f44c0197e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.613+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c01983b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 -- 172.21.15.40:0/2438361839 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c00fe9d0 msgr2=0x7f44c0197e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c00fe9d0 0x7f44c0197e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 -- 172.21.15.40:0/2438361839 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 msgr2=0x7f44c01983b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c01983b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 -- 172.21.15.40:0/2438361839 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f44b8009d00 con 0x7f44c01049e0 2023-07-14T11:13:09.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f44b8009fd0 tx=0x7f44b800bce0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.614+0000 7f44b77fe700 1 -- 172.21.15.40:0/2438361839 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b800e040 con 0x7f44c01049e0 2023-07-14T11:13:09.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.615+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f44c019d400 con 0x7f44c01049e0 2023-07-14T11:13:09.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.615+0000 7f44b77fe700 1 -- 172.21.15.40:0/2438361839 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44b80094c0 con 0x7f44c01049e0 2023-07-14T11:13:09.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.615+0000 7f44b77fe700 1 -- 172.21.15.40:0/2438361839 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b8004520 con 0x7f44c01049e0 2023-07-14T11:13:09.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.615+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44c01a4ca0 con 0x7f44c01049e0 2023-07-14T11:13:09.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.619+0000 7f44b77fe700 1 -- 172.21.15.40:0/2438361839 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f44b8004680 con 0x7f44c01049e0 2023-07-14T11:13:09.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.620+0000 7f44b77fe700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 0x7f44ac0a3910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.620+0000 7f44b77fe700 1 -- 172.21.15.40:0/2438361839 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f44b80ce460 con 0x7f44c01049e0 2023-07-14T11:13:09.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44dc1e8b80 1 Processor -- start 2023-07-14T11:13:09.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44c659b700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 0x7f44ac0a3910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44dc1e8b80 1 -- start start 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd276a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd27be0 0x562d1bd3ada0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.621+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3d6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bc2dcb0 con 0x562d1bd27be0 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bc80a40 con 0x562d1bd3b2e0 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bc79ed0 con 0x562d1bd272d0 2023-07-14T11:13:09.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c659b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd276a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c659b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd276a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50906/0 (socket says 172.21.15.40:50906) 2023-07-14T11:13:09.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3d6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c659b700 1 -- 172.21.15.40:0/1562093863 learned_addr learned my addr 172.21.15.40:0/1562093863 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd27be0 0x562d1bd3ada0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 -- 172.21.15.40:0/1562093863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 msgr2=0x562d1bd276a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd276a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 -- 172.21.15.40:0/1562093863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd27be0 msgr2=0x562d1bd3ada0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd27be0 0x562d1bd3ada0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.622+0000 7f44c6d9c700 1 -- 172.21.15.40:0/1562093863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562d1baa7de0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.623+0000 7f44c659b700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 0x7f44ac0a3910 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f44c00ffb10 tx=0x7f44bc009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.623+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3d6c0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f44b80cb700 tx=0x7f44b800b3b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7574a924a64852c6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.623+0000 7f44b4ff9700 1 -- 172.21.15.40:0/1562093863 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b8004c60 con 0x562d1bd3b2e0 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.623+0000 7f44b4ff9700 1 -- 172.21.15.40:0/1562093863 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44b809b040 con 0x562d1bd3b2e0 2023-07-14T11:13:09.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.623+0000 7f44b4ff9700 1 -- 172.21.15.40:0/1562093863 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b809b4e0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/1562093863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 msgr2=0x562d1bd3d6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3d6c0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f44b80cb700 tx=0x7f44b800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/1562093863 shutdown_connections 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3d6c0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd276a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/1562093863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd27be0 0x562d1bd3ada0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/1562093863 >> 172.21.15.40:0/1562093863 conn(0x562d1bd1dda0 msgr2=0x562d1bd236a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/1562093863 shutdown_connections 2023-07-14T11:13:09.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.624+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/1562093863 wait complete. 2023-07-14T11:13:09.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 -- start start 2023-07-14T11:13:09.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 Processor -- start 2023-07-14T11:13:09.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 0x562d1bd34360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd348a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2f630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bc19ac0 con 0x562d1bd272d0 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bc197c0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bafd920 con 0x562d1bd27be0 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.625+0000 7f44dc1e8b80 10 client.0 _ll_register_callbacks cb 0x562d1bc17760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c659b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 0x562d1bd34360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c659b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 0x562d1bd34360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45206/0 (socket says 172.21.15.40:45206) 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c659b700 1 -- 172.21.15.40:0/233160745 learned_addr learned my addr 172.21.15.40:0/233160745 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2f630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd348a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 -- 172.21.15.40:0/233160745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 msgr2=0x562d1bd348a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd348a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 -- 172.21.15.40:0/233160745 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 msgr2=0x562d1bd34360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 0x562d1bd34360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 -- 172.21.15.40:0/233160745 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f44c01a4ca0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd348a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2f630 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f44b80cb700 tx=0x7f44b80099e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2fd12bb15bd32a28 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.626+0000 7f44a9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b8004c60 con 0x562d1bd3b2e0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f44b809f070 con 0x562d1bd3b2e0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562d1bbd34f0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.634 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f44b809b4b0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44dc1e8b80 10 client.26021 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x562d1ba90c60 con 0x562d1bd3b2e0 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.627+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44b80c9070 con 0x562d1bd3b2e0 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f44b80cb9e0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 10 client.26021 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 10 client.26021 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 20 client.26021 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:09.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bd52f20 0x562d1bd55300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 4 client.26021 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:09.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562d1bc72eb0 con 0x562d1bd52f20 2023-07-14T11:13:09.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.628+0000 7f44c659b700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bd52f20 0x562d1bd55300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.629+0000 7f44c659b700 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bd52f20 0x562d1bd55300 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b73703269c8665c0 server_cookie=2d334d7ad17b0838 in_seq=0 out_seq=0 2023-07-14T11:13:09.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.629+0000 7f44a9ffb700 10 client.26021 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:09.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44a9ffb700 1 -- 172.21.15.40:0/233160745 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562d1bc72eb0 con 0x562d1bd52f20 2023-07-14T11:13:09.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44a9ffb700 10 client.26021 handle_command_reply: tid=0 2023-07-14T11:13:09.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 client.26021 shutdown 2023-07-14T11:13:09.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 msgr2=0x562d1bd2f630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2f630 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f44b80cb700 tx=0x7f44b80099e0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 shutdown_connections 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bd52f20 0x562d1bd55300 crc :-1 s=READY pgs=65 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2f630 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd272d0 0x562d1bd34360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/233160745 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd348a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 >> 172.21.15.40:0/233160745 conn(0x562d1bd1dda0 msgr2=0x562d1bd23290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.630+0000 7f44a9ffb700 0 client.26021 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:09.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.631+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 shutdown_connections 2023-07-14T11:13:09.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.631+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/233160745 wait complete. 2023-07-14T11:13:09.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.631+0000 7f44dc1e8b80 20 client.26021 trim_cache size 0 max 16384 2023-07-14T11:13:09.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.649+0000 7f44dc1e8b80 1 Processor -- start 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.649+0000 7f44dc1e8b80 1 -- start start 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd272d0 0x562d1bd2f5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3c6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd1fec0 con 0x562d1bd3b2e0 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd28540 con 0x562d1bd272d0 2023-07-14T11:13:09.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd30c10 con 0x562d1bd27be0 2023-07-14T11:13:09.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44c5d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44c5d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50926/0 (socket says 172.21.15.40:50926) 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44c5d9a700 1 -- 172.21.15.40:0/2699907903 learned_addr learned my addr 172.21.15.40:0/2699907903 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.650+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3c6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c659b700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd272d0 0x562d1bd2f5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 -- 172.21.15.40:0/2699907903 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd272d0 msgr2=0x562d1bd2f5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd272d0 0x562d1bd2f5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 -- 172.21.15.40:0/2699907903 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 msgr2=0x562d1bd3c6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3c6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 -- 172.21.15.40:0/2699907903 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562d1bd1f5e0 con 0x562d1bd27be0 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3c6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f44b0009c60 tx=0x7f44b0009f70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e0468db86eedd75f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.651+0000 7f44b57fa700 1 -- 172.21.15.40:0/2699907903 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b0010bc0 con 0x562d1bd27be0 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.652+0000 7f44b57fa700 1 -- 172.21.15.40:0/2699907903 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44b0018040 con 0x562d1bd27be0 2023-07-14T11:13:09.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.652+0000 7f44b57fa700 1 -- 172.21.15.40:0/2699907903 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44b000f620 con 0x562d1bd27be0 2023-07-14T11:13:09.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.652+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2699907903 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 msgr2=0x562d1bd2e820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.652+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f44b0009c60 tx=0x7f44b0009f70 comp rx=0 tx=0).stop 2023-07-14T11:13:09.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.652+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2699907903 shutdown_connections 2023-07-14T11:13:09.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd3c6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd272d0 0x562d1bd2f5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2699907903 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd27be0 0x562d1bd2e820 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2699907903 >> 172.21.15.40:0/2699907903 conn(0x562d1bd1dda0 msgr2=0x562d1bd2eed0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2699907903 shutdown_connections 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2699907903 wait complete. 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 -- start start 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.653+0000 7f44dc1e8b80 1 Processor -- start 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd27be0 0x562d1bd39ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2c790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd2ce90 con 0x562d1bd3b2e0 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd2d000 con 0x562d1bd27be0 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562d1bd2d1c0 con 0x562d1bd272d0 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44dc1e8b80 10 client.0 _ll_register_callbacks cb 0x7f44b8009c30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44c659b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44c659b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50938/0 (socket says 172.21.15.40:50938) 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44c5d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd27be0 0x562d1bd39ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.654+0000 7f44c659b700 1 -- 172.21.15.40:0/2978942711 learned_addr learned my addr 172.21.15.40:0/2978942711 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2c790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 -- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd27be0 msgr2=0x562d1bd39ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd27be0 0x562d1bd39ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 -- 172.21.15.40:0/2978942711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 msgr2=0x562d1bd2c790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2c790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 -- 172.21.15.40:0/2978942711 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562d1ba90c60 con 0x562d1bd272d0 2023-07-14T11:13:09.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c6d9c700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2c790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:13:09.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44c659b700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f44bc02acb0 tx=0x7f44bc0047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=51c72dd490085768 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:09.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.655+0000 7f44a9ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:09.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f44bc021040 con 0x562d1bd272d0 2023-07-14T11:13:09.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f44bc004020 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f44b809b040 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f44bc02a5e0 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44dc1e8b80 10 client.26014 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x7f44b0009910 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f44b809b040 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.656+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f44bc0037a0 con 0x562d1bd272d0 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 10 client.26014 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:09.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 10 client.26014 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 20 client.26014 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bda7fb0 0x562d1bdaa3c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 4 client.26014 mds_command: new command op to 24569 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x562d1bc35050 con 0x562d1bda7fb0 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.657+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bda7fb0 0x562d1bdaa3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44c5d9a700 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bda7fb0 0x562d1bdaa3c0 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=95e8cb297544e5c server_cookie=3d41cc9160b458b1 in_seq=0 out_seq=0 2023-07-14T11:13:09.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44a9ffb700 10 client.26014 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44a9ffb700 1 -- 172.21.15.40:0/2978942711 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x562d1bc35050 con 0x562d1bda7fb0 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44a9ffb700 10 client.26014 handle_command_reply: tid=0 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 client.26014 shutdown 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 msgr2=0x562d1bd399a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f44bc02acb0 tx=0x7f44bc0047c0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 shutdown_connections 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562d1bda7fb0 0x562d1bdaa3c0 crc :-1 s=READY pgs=66 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562d1bd3b2e0 0x562d1bd2c790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562d1bd272d0 0x562d1bd399a0 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 --2- 172.21.15.40:0/2978942711 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562d1bd27be0 0x562d1bd39ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44a9ffb700 0 client.26014 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.658+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 >> 172.21.15.40:0/2978942711 conn(0x562d1bd1dda0 msgr2=0x562d1bd29290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 shutdown_connections 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44dc1e8b80 1 -- 172.21.15.40:0/2978942711 wait complete. 2023-07-14T11:13:09.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44dc1e8b80 20 client.26014 trim_cache size 0 max 16384 2023-07-14T11:13:09.668 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:13:09.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 msgr2=0x7f44ac0a3910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 0x7f44ac0a3910 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f44c00ffb10 tx=0x7f44bc009260 comp rx=0 tx=0).stop 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 msgr2=0x7f44c019c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.659+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 crypto rx=0x7f44b8009fd0 tx=0x7f44b800bce0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 shutdown_connections 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f44ac0a1460 0x7f44ac0a3910 secure :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f44c00ffb10 tx=0x7f44bc009260 comp rx=0 tx=0).stop 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f44c01049e0 0x7f44c019c810 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f44c00fe9d0 0x7f44c0197e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 --2- 172.21.15.40:0/2438361839 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f44c00ff380 0x7f44c01983b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:09.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 >> 172.21.15.40:0/2438361839 conn(0x7f44c00fa4b0 msgr2=0x7f44c010c110 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:09.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 shutdown_connections 2023-07-14T11:13:09.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:09.660+0000 7f44c87ff700 1 -- 172.21.15.40:0/2438361839 wait complete. 2023-07-14T11:13:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:09 smithi130 ceph-mon[129405]: pgmap v33: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 210 op/s 2023-07-14T11:13:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:09 smithi040 ceph-mon[97954]: pgmap v33: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 210 op/s 2023-07-14T11:13:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:09 smithi045 ceph-mon[128791]: pgmap v33: 129 pgs: 129 active+clean; 2.3 GiB data, 5.0 GiB used, 1.0 TiB / 1.0 TiB avail; 32 KiB/s rd, 24 MiB/s wr, 210 op/s 2023-07-14T11:13:10.685 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2023-07-14T11:13:10.685 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2023-07-14T11:13:10.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/2905000009 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40ff380 msgr2=0x7f4cf410b810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/2905000009 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40ff380 0x7f4cf410b810 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f4cec00b0a0 tx=0x7f4cec00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/2905000009 shutdown_connections 2023-07-14T11:13:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/2905000009 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40ff380 0x7f4cf410b810 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/2905000009 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf41049e0 0x7f4cf4104db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/2905000009 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf40fee40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.946+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/2905000009 >> 172.21.15.40:0/2905000009 conn(0x7f4cf40fa4f0 msgr2=0x7f4cf40fc900 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.947+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/2905000009 shutdown_connections 2023-07-14T11:13:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.947+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/2905000009 wait complete. 2023-07-14T11:13:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.948+0000 7f4cfb5ff700 1 Processor -- start 2023-07-14T11:13:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.948+0000 7f4cfb5ff700 1 -- start start 2023-07-14T11:13:10.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf40ff380 0x7f4cf4198550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf41049e0 0x7f4cf419c9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf419cff0 con 0x7f4cf40ff380 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf419d160 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cfb5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf419d320 con 0x7f4cf41049e0 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cf8b9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf40ff380 0x7f4cf4198550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49618/0 (socket says 172.21.15.40:49618) 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cf939b700 1 -- 172.21.15.40:0/1324604661 learned_addr learned my addr 172.21.15.40:0/1324604661 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.949+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf41049e0 0x7f4cf419c9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 -- 172.21.15.40:0/1324604661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf41049e0 msgr2=0x7f4cf419c9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:10.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf41049e0 0x7f4cf419c9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 -- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf40ff380 msgr2=0x7f4cf4198550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf40ff380 0x7f4cf4198550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 -- 172.21.15.40:0/1324604661 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cec009d00 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cf939b700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f4cf000b3c0 tx=0x7f4cf0009ab0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cea7fc700 1 -- 172.21.15.40:0/1324604661 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cf0018070 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cea7fc700 1 -- 172.21.15.40:0/1324604661 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4cf00044e0 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.950+0000 7f4cea7fc700 1 -- 172.21.15.40:0/1324604661 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cf0010040 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.951+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cf419d600 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.951+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cf41a4e40 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.955+0000 7f4cea7fc700 1 -- 172.21.15.40:0/1324604661 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7f4cf0003f20 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.957+0000 7f4d0ee5fb80 1 Processor -- start 2023-07-14T11:13:10.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.957+0000 7f4d0ee5fb80 1 -- start start 2023-07-14T11:13:10.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.957+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.957+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1c60 0x55e52eadc610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eadef30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52e9d7d30 con 0x55e52ead1c60 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52ea2aac0 con 0x55e52eadcb50 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52ea23f50 con 0x55e52ead1350 2023-07-14T11:13:10.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cea7fc700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 0x7f4ce00a39e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:50972/0 (socket says 172.21.15.40:50972) 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/2071079552 learned_addr learned my addr 172.21.15.40:0/2071079552 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf939b700 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1c60 0x55e52eadc610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cea7fc700 1 -- 172.21.15.40:0/1324604661 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4cf00cc900 con 0x7f4cf40fe9d0 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eadef30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 0x7f4ce00a39e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/2071079552 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 msgr2=0x55e52eadef30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eadef30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/2071079552 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1c60 msgr2=0x55e52eadc610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1c60 0x55e52eadc610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.958+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/2071079552 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e52e851e60 con 0x55e52ead1350 2023-07-14T11:13:10.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.959+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f4cec015000 tx=0x7f4cec00bd30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f0e0a9b897ce420c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.959+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 0x7f4ce00a39e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f4cec024000 tx=0x7f4cec027000 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.959+0000 7f4cdf7fe700 1 -- 172.21.15.40:0/2071079552 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec00e020 con 0x55e52ead1350 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.959+0000 7f4cdf7fe700 1 -- 172.21.15.40:0/2071079552 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4cec00bab0 con 0x55e52ead1350 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.959+0000 7f4cdf7fe700 1 -- 172.21.15.40:0/2071079552 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec005040 con 0x55e52ead1350 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2071079552 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 msgr2=0x55e52ead1720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f4cec015000 tx=0x7f4cec00bd30 comp rx=0 tx=0).stop 2023-07-14T11:13:10.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2071079552 shutdown_connections 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eadef30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1350 0x55e52ead1720 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2071079552 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1c60 0x55e52eadc610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2071079552 >> 172.21.15.40:0/2071079552 conn(0x55e52eac7e20 msgr2=0x55e52eacd720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2071079552 shutdown_connections 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.960+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2071079552 wait complete. 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.961+0000 7f4d0ee5fb80 1 -- start start 2023-07-14T11:13:10.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.961+0000 7f4d0ee5fb80 1 Processor -- start 2023-07-14T11:13:10.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52ead9e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eada350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52e9c3b40 con 0x55e52eadcb50 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52e9c3840 con 0x55e52ead1350 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52e8a79a0 con 0x55e52ead1c60 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4d0ee5fb80 10 client.0 _ll_register_callbacks cb 0x55e52e9c17e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4cf8b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52ead9e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49636/0 (socket says 172.21.15.40:49636) 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.962+0000 7f4cf939b700 1 -- 172.21.15.40:0/675036995 learned_addr learned my addr 172.21.15.40:0/675036995 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eada350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 msgr2=0x55e52ead98d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/675036995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 msgr2=0x55e52eada350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.970 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eada350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/675036995 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cf410c110 con 0x55e52ead1c60 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf939b700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52ead9e10 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4011540 tx=0x7f4ce4009f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=600d562196efb4f4 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.963+0000 7f4cdcff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ce4011e60 con 0x55e52ead1c60 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4ce4014650 con 0x55e52ead1c60 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e52ea1cf30 con 0x55e52ead1c60 2023-07-14T11:13:10.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4ce4014810 con 0x55e52ead1c60 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4d0ee5fb80 10 client.26023 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.964+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55e52e83ace0 con 0x55e52ead1c60 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e52ea1cf30 con 0x55e52ead1c60 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f4ce401b020 con 0x55e52ead1c60 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 10 client.26023 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 10 client.26023 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 20 client.26023 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:10.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eaf4d70 0x55e52eaf7180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 4 client.26023 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:10.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e52e9df0d0 con 0x55e52eaf4d70 2023-07-14T11:13:10.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.965+0000 7f4cf939b700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eaf4d70 0x55e52eaf7180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.966+0000 7f4cf939b700 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eaf4d70 0x55e52eaf7180 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c8d83c25354bb033 server_cookie=3d3b9665b1306d98 in_seq=0 out_seq=0 2023-07-14T11:13:10.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.966+0000 7f4cdcff9700 10 client.26023 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:10.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.967+0000 7f4cdcff9700 1 -- 172.21.15.40:0/675036995 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55e52e9df0d0 con 0x55e52eaf4d70 2023-07-14T11:13:10.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.967+0000 7f4cdcff9700 10 client.26023 handle_command_reply: tid=0 2023-07-14T11:13:10.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.967+0000 7f4d0ee5fb80 1 client.26023 shutdown 2023-07-14T11:13:10.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.967+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 msgr2=0x55e52ead9e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:10.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.967+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52ead9e10 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4011540 tx=0x7f4ce4009f90 comp rx=0 tx=0).stop 2023-07-14T11:13:10.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 shutdown_connections 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eaf4d70 0x55e52eaf7180 crc :-1 s=READY pgs=67 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eada350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52ead98d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4cdcff9700 0 client.26023 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/675036995 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52ead9e10 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 >> 172.21.15.40:0/675036995 conn(0x55e52eac7e20 msgr2=0x55e52eacd2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:10.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.968+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 shutdown_connections 2023-07-14T11:13:10.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.969+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/675036995 wait complete. 2023-07-14T11:13:10.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.969+0000 7f4d0ee5fb80 20 client.26023 trim_cache size 0 max 16384 2023-07-14T11:13:10.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 Processor -- start 2023-07-14T11:13:10.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 -- start start 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eacd250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eacd790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52ead2eb0 con 0x55e52ead1350 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52ead25c0 con 0x55e52eadcb50 2023-07-14T11:13:10.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52eace3b0 con 0x55e52ead1c60 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eacd790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4cf8b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eacd250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4cf939b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45238/0 (socket says 172.21.15.40:45238) 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf939b700 1 -- 172.21.15.40:0/2840080257 learned_addr learned my addr 172.21.15.40:0/2840080257 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.993+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eacd790 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49648/0 (socket says 172.21.15.40:49648) 2023-07-14T11:13:10.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/2840080257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 msgr2=0x55e52eacd790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eacd790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/2840080257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 msgr2=0x55e52eaebe50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 -- 172.21.15.40:0/2840080257 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e52e8782d0 con 0x55e52ead1c60 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf939b700 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cf8b9a700 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eacd250 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4011b30 tx=0x7f4ce4006ed0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6fa603c1b9ead533 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cdffff700 1 -- 172.21.15.40:0/2840080257 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ce4009280 con 0x55e52ead1c60 2023-07-14T11:13:10.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cdffff700 1 -- 172.21.15.40:0/2840080257 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4ce4011e60 con 0x55e52ead1c60 2023-07-14T11:13:11.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.994+0000 7f4cdffff700 1 -- 172.21.15.40:0/2840080257 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4ce401b720 con 0x55e52ead1c60 2023-07-14T11:13:11.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2840080257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 msgr2=0x55e52eacd250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eacd250 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7f4ce4011b30 tx=0x7f4ce4006ed0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2840080257 shutdown_connections 2023-07-14T11:13:11.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52eadcb50 0x55e52eacd790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52ead1350 0x55e52eaebe50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/2840080257 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eacd250 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2840080257 >> 172.21.15.40:0/2840080257 conn(0x55e52eac7e20 msgr2=0x55e52ead4bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.995+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2840080257 shutdown_connections 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.996+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/2840080257 wait complete. 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.996+0000 7f4d0ee5fb80 1 -- start start 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 Processor -- start 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52eb42b70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eb430b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52eb3d4f0 con 0x55e52eadcb50 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52eb3d660 con 0x55e52ead1350 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e52eb3d820 con 0x55e52ead1c60 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.997+0000 7f4d0ee5fb80 10 client.0 _ll_register_callbacks cb 0x7f4ce4009b00 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45250/0 (socket says 172.21.15.40:45250) 2023-07-14T11:13:11.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/154110435 learned_addr learned my addr 172.21.15.40:0/154110435 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf939b700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52eb42b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/154110435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 msgr2=0x55e52eb430b0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eb430b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 msgr2=0x55e52eb42b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52eb42b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf9b9c700 1 -- 172.21.15.40:0/154110435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cf41a4e40 con 0x55e52eadcb50 2023-07-14T11:13:11.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.998+0000 7f4cf939b700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52eb42b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:11.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4cf9b9c700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f4cec0095e0 tx=0x7f4cec00bba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b407445d42186162 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4cdcff9700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4cec00e040 con 0x55e52eadcb50 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f4cec026070 con 0x55e52eadcb50 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e52e97d570 con 0x55e52eadcb50 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4cec006080 con 0x55e52eadcb50 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:10.999+0000 7f4d0ee5fb80 10 client.16044 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:11.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55e52e83ace0 con 0x55e52eadcb50 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e52e97d570 con 0x55e52eadcb50 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f4cec019070 con 0x55e52eadcb50 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 10 client.16044 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 10 client.16044 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 20 client.16044 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eb4a810 0x55e52eb4cc20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 4 client.16044 mds_command: new command op to 24569 tid=0[{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2023-07-14T11:13:11.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x55e52e872ff0 con 0x55e52eb4a810 2023-07-14T11:13:11.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.000+0000 7f4cf939b700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eb4a810 0x55e52eb4cc20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.001+0000 7f4cdcff9700 10 client.16044 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.001+0000 7f4cf939b700 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eb4a810 0x55e52eb4cc20 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=43f052139d79006c server_cookie=42753ffb35a33a52 in_seq=0 out_seq=0 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.003+0000 7f4cdcff9700 1 -- 172.21.15.40:0/154110435 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x55e52e872ff0 con 0x55e52eb4a810 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.003+0000 7f4cdcff9700 10 client.16044 handle_command_reply: tid=0 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.003+0000 7f4d0ee5fb80 1 client.16044 shutdown 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 msgr2=0x55e52eb3cf10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 crypto rx=0x7f4cec0095e0 tx=0x7f4cec00bba0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 shutdown_connections 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e52eb4a810 0x55e52eb4cc20 crc :-1 s=READY pgs=68 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e52eadcb50 0x55e52eb3cf10 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e52ead1350 0x55e52eb42b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4cdcff9700 0 client.16044 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 --2- 172.21.15.40:0/154110435 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e52ead1c60 0x55e52eb430b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 >> 172.21.15.40:0/154110435 conn(0x55e52eac7e20 msgr2=0x55e52eacd1e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 shutdown_connections 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.004+0000 7f4d0ee5fb80 1 -- 172.21.15.40:0/154110435 wait complete. 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.005+0000 7f4d0ee5fb80 20 client.16044 trim_cache size 0 max 16384 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stdout: "return_code": 0, 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_tag": "88bd81f4-489e-47f0-a82d-3ec87899c170", 2023-07-14T11:13:11.011 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": "asynchronous" 2023-07-14T11:13:11.012 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:13:11.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.005+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 msgr2=0x7f4ce00a39e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.005+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 0x7f4ce00a39e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f4cec024000 tx=0x7f4cec027000 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.005+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 msgr2=0x7f4cf4198010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f4cf000b3c0 tx=0x7f4cf0009ab0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 shutdown_connections 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4ce00a1530 0x7f4ce00a39e0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4cf41049e0 0x7f4cf419c9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4cf40fe9d0 0x7f4cf4198010 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 --2- 172.21.15.40:0/1324604661 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4cf40ff380 0x7f4cf4198550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 >> 172.21.15.40:0/1324604661 conn(0x7f4cf40fa4f0 msgr2=0x7f4cf40fb930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 shutdown_connections 2023-07-14T11:13:11.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.006+0000 7f4cfb5ff700 1 -- 172.21.15.40:0/1324604661 wait complete. 2023-07-14T11:13:11.035 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:13:11.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.299+0000 7fa7e59ff700 1 -- 172.21.15.40:0/1758044587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 msgr2=0x7fa7e010e760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.299+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/1758044587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e010e760 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fa7d8009f90 tx=0x7fa7d800b330 comp rx=0 tx=0).stop 2023-07-14T11:13:11.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 -- 172.21.15.40:0/1758044587 shutdown_connections 2023-07-14T11:13:11.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/1758044587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e010e760 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/1758044587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0072c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/1758044587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e010be40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 -- 172.21.15.40:0/1758044587 >> 172.21.15.40:0/1758044587 conn(0x7fa7e00fb5e0 msgr2=0x7fa7e00fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 -- 172.21.15.40:0/1758044587 shutdown_connections 2023-07-14T11:13:11.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.300+0000 7fa7e59ff700 1 -- 172.21.15.40:0/1758044587 wait complete. 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.301+0000 7fa7e59ff700 1 Processor -- start 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.301+0000 7fa7e59ff700 1 -- start start 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e01984b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e019c910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7e019cf50 con 0x7fa7e0073160 2023-07-14T11:13:11.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7e019d0c0 con 0x7fa7e010c380 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7e59ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa7e019d280 con 0x7fa7e0072850 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:51006/0 (socket says 172.21.15.40:51006) 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 -- 172.21.15.40:0/2857529251 learned_addr learned my addr 172.21.15.40:0/2857529251 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e01984b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e019c910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 -- 172.21.15.40:0/2857529251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 msgr2=0x7fa7e019c910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e019c910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 -- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 msgr2=0x7fa7e01984b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e01984b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7deffd700 1 -- 172.21.15.40:0/2857529251 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa7d8009c40 con 0x7fa7e0072850 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.302+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e01984b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fa7d000e9c0 tx=0x7fa7d000ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7e49fd700 1 -- 172.21.15.40:0/2857529251 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d0004a30 con 0x7fa7e0072850 2023-07-14T11:13:11.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa7e019d560 con 0x7fa7e0072850 2023-07-14T11:13:11.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7e49fd700 1 -- 172.21.15.40:0/2857529251 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa7d000cc80 con 0x7fa7e0072850 2023-07-14T11:13:11.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7e49fd700 1 -- 172.21.15.40:0/2857529251 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d00054e0 con 0x7fa7e0072850 2023-07-14T11:13:11.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.303+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7e01a4da0 con 0x7fa7e0072850 2023-07-14T11:13:11.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.307+0000 7fa7e49fd700 1 -- 172.21.15.40:0/2857529251 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 23) v1 ==== 135948+0+0 (secure 0 0 0) 0x7fa7d0004b90 con 0x7fa7e0072850 2023-07-14T11:13:11.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 Processor -- start 2023-07-14T11:13:11.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 -- start start 2023-07-14T11:13:11.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e366a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e43eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3d3ccb0 con 0x55bfe3e362d0 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3d8fa40 con 0x55bfe3e36be0 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3d88ed0 con 0x55bfe3e41ad0 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7de7fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e43eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.309+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49690/0 (socket says 172.21.15.40:49690) 2023-07-14T11:13:11.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7de7fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e43eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:51012/0 (socket says 172.21.15.40:51012) 2023-07-14T11:13:11.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 -- 172.21.15.40:0/2137796658 learned_addr learned my addr 172.21.15.40:0/2137796658 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e366a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7e49fd700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 0x7fa7c80a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 -- 172.21.15.40:0/2137796658 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 msgr2=0x55bfe3e43eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e43eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 -- 172.21.15.40:0/2137796658 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 msgr2=0x55bfe3e366a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e366a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 -- 172.21.15.40:0/2137796658 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfe3bb6de0 con 0x55bfe3e36be0 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7e49fd700 1 -- 172.21.15.40:0/2857529251 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fa7d0014070 con 0x7fa7e0072850 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 0x7fa7c80a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fa7d00ca610 tx=0x7fa7d0065370 comp rx=0 tx=0).ready entity=mon.2 client_cookie=33d68477c1cc6b29 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.310+0000 7fa7c5ffb700 1 -- 172.21.15.40:0/2137796658 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d0099cf0 con 0x55bfe3e36be0 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.311+0000 7fa7c5ffb700 1 -- 172.21.15.40:0/2137796658 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa7d0099a60 con 0x55bfe3e36be0 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.311+0000 7fa7c5ffb700 1 -- 172.21.15.40:0/2137796658 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d00d0db0 con 0x55bfe3e36be0 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.311+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2137796658 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 msgr2=0x55bfe3e41590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.311+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fa7d00ca610 tx=0x7fa7d0065370 comp rx=0 tx=0).stop 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.311+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 0x7fa7c80a3890 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fa7d8009790 tx=0x7fa7d8009920 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2137796658 shutdown_connections 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e43eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e366a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2137796658 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41590 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2137796658 >> 172.21.15.40:0/2137796658 conn(0x55bfe3e2cda0 msgr2=0x55bfe3e326a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2137796658 shutdown_connections 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2137796658 wait complete. 2023-07-14T11:13:11.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.312+0000 7fa7f9265b80 1 -- start start 2023-07-14T11:13:11.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.313+0000 7fa7f9265b80 1 Processor -- start 2023-07-14T11:13:11.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.313+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.313+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3e700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3ed30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3d28ac0 con 0x55bfe3e36be0 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3d287c0 con 0x55bfe3e362d0 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3c0c920 con 0x55bfe3e41ad0 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7f9265b80 10 client.0 _ll_register_callbacks cb 0x55bfe3d26760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49698/0 (socket says 172.21.15.40:49698) 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 -- 172.21.15.40:0/2719397203 learned_addr learned my addr 172.21.15.40:0/2719397203 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 -- 172.21.15.40:0/2719397203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 msgr2=0x55bfe3e3ed30 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:13:11.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3ed30 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 -- 172.21.15.40:0/2719397203 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 msgr2=0x55bfe3e3e700 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3e700 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 -- 172.21.15.40:0/2719397203 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7e01a4da0 con 0x55bfe3e362d0 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7deffd700 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096990 tx=0x7fa7d0099880 comp rx=0 tx=0).ready entity=mon.2 client_cookie=34cf44b954158397 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.314+0000 7fa7beffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d006d360 con 0x55bfe3e362d0 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fa7d000cdf0 con 0x55bfe3e362d0 2023-07-14T11:13:11.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55bfe3ce24f0 con 0x55bfe3e362d0 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa7d006d540 con 0x55bfe3e362d0 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7f9265b80 10 client.26057 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.315+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55bfe3b9fc60 con 0x55bfe3e362d0 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55bfe3ce24f0 con 0x55bfe3e362d0 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7fa7d00a1ee0 con 0x55bfe3e362d0 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 10 client.26057 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 10 client.26057 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:11.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 20 client.26057 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3e50ff0 0x55bfe3e53400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 4 client.26057 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55bfe3d81eb0 con 0x55bfe3e50ff0 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.316+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3e50ff0 0x55bfe3e53400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.317+0000 7fa7beffd700 10 client.26057 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.317+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3e50ff0 0x55bfe3e53400 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=bd00a4c5f683c24b server_cookie=671d739f60af7af9 in_seq=0 out_seq=0 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7beffd700 1 -- 172.21.15.40:0/2719397203 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55bfe3d81eb0 con 0x55bfe3e50ff0 2023-07-14T11:13:11.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7beffd700 10 client.26057 handle_command_reply: tid=0 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 client.26057 shutdown 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 msgr2=0x55bfe3e414f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096990 tx=0x7fa7d0099880 comp rx=0 tx=0).stop 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 shutdown_connections 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3e50ff0 0x55bfe3e53400 crc :-1 s=READY pgs=69 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3ed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e414f0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/2719397203 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7beffd700 0 client.26057 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 >> 172.21.15.40:0/2719397203 conn(0x55bfe3e2cda0 msgr2=0x55bfe3e32290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 shutdown_connections 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 1 -- 172.21.15.40:0/2719397203 wait complete. 2023-07-14T11:13:11.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.318+0000 7fa7f9265b80 20 client.26057 trim_cache size 0 max 16384 2023-07-14T11:13:11.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 Processor -- start 2023-07-14T11:13:11.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 -- start start 2023-07-14T11:13:11.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e489e0 con 0x55bfe3e362d0 2023-07-14T11:13:11.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e53400 con 0x55bfe3e41ad0 2023-07-14T11:13:11.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.340+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e2e310 con 0x55bfe3e36be0 2023-07-14T11:13:11.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7df7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7df7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49712/0 (socket says 172.21.15.40:49712) 2023-07-14T11:13:11.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7df7fe700 1 -- 172.21.15.40:0/510835977 learned_addr learned my addr 172.21.15.40:0/510835977 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 -- 172.21.15.40:0/510835977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e36be0 msgr2=0x55bfe3e41020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 -- 172.21.15.40:0/510835977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 msgr2=0x55bfe3e39a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.341+0000 7fa7deffd700 1 -- 172.21.15.40:0/510835977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55bfe3bdd250 con 0x55bfe3e362d0 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.342+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:11.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.342+0000 7fa7deffd700 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484a0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096420 tx=0x7fa7d00984d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1063adb6beb52e5d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.342+0000 7fa7c67fc700 1 -- 172.21.15.40:0/510835977 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d009a180 con 0x55bfe3e362d0 2023-07-14T11:13:11.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.342+0000 7fa7c67fc700 1 -- 172.21.15.40:0/510835977 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa7d00d4070 con 0x55bfe3e362d0 2023-07-14T11:13:11.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.342+0000 7fa7c67fc700 1 -- 172.21.15.40:0/510835977 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d00a2370 con 0x55bfe3e362d0 2023-07-14T11:13:11.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 -- 172.21.15.40:0/510835977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 msgr2=0x55bfe3e484a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484a0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096420 tx=0x7fa7d00984d0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 -- 172.21.15.40:0/510835977 shutdown_connections 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e39a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484a0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/510835977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e36be0 0x55bfe3e41020 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 -- 172.21.15.40:0/510835977 >> 172.21.15.40:0/510835977 conn(0x55bfe3e2cda0 msgr2=0x55bfe3e30e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 -- 172.21.15.40:0/510835977 shutdown_connections 2023-07-14T11:13:11.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.343+0000 7fa7f9265b80 1 -- 172.21.15.40:0/510835977 wait complete. 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.344+0000 7fa7f9265b80 1 -- start start 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.344+0000 7fa7f9265b80 1 Processor -- start 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3b060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e3b640 con 0x55bfe3e36be0 2023-07-14T11:13:11.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e3b7b0 con 0x55bfe3e362d0 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55bfe3e3b970 con 0x55bfe3e41ad0 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7f9265b80 10 client.0 _ll_register_callbacks cb 0x7fa7d4005ec0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7de7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7deffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7de7fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45302/0 (socket says 172.21.15.40:45302) 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7de7fc700 1 -- 172.21.15.40:0/3079161628 learned_addr learned my addr 172.21.15.40:0/3079161628 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:11.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7df7fe700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3b060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7deffd700 1 -- 172.21.15.40:0/3079161628 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 msgr2=0x55bfe3e3b060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.345+0000 7fa7deffd700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3b060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7deffd700 1 -- 172.21.15.40:0/3079161628 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 msgr2=0x55bfe3e3ab20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7deffd700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7deffd700 1 -- 172.21.15.40:0/3079161628 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa7e010f3a0 con 0x55bfe3e362d0 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:11.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7deffd700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096420 tx=0x7fa7d0096020 comp rx=0 tx=0).ready entity=mon.2 client_cookie=59479be1f3791781 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:11.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7beffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:13:11.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa7d009a180 con 0x55bfe3e362d0 2023-07-14T11:13:11.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fa7d000cdf0 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa7d0099a60 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=30) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa7d0099cf0 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7f9265b80 10 client.26063 fetch_fsmap learned FSMap version 30 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.346+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=30,monmap=4+}) v3 -- 0x55bfe3b9fc60 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa7d0099a60 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 30) v1 ==== 7486+0+0 (secure 0 0 0) 0x7fa7d0016c00 con 0x55bfe3e362d0 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 10 client.26063 fetch_fsmap finished waiting for FSMap version 30 2023-07-14T11:13:11.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 10 client.26063 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 20 client.26063 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3ea5f50 0x55bfe3ea8330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 4 client.26063 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55bfe3d44050 con 0x55bfe3ea5f50 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.347+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3ea5f50 0x55bfe3ea8330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7de7fc700 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3ea5f50 0x55bfe3ea8330 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5d8dba95917417ec server_cookie=780413f5476628d1 in_seq=0 out_seq=0 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7beffd700 10 client.26063 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7beffd700 1 -- 172.21.15.40:0/3079161628 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55bfe3d44050 con 0x55bfe3ea5f50 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7beffd700 10 client.26063 handle_command_reply: tid=0 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 client.26063 shutdown 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 msgr2=0x55bfe3e484b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fa7d0096420 tx=0x7fa7d0096020 comp rx=0 tx=0).stop 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 shutdown_connections 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55bfe3ea5f50 0x55bfe3ea8330 crc :-1 s=READY pgs=70 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55bfe3e41ad0 0x55bfe3e3b060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55bfe3e362d0 0x55bfe3e484b0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 --2- 172.21.15.40:0/3079161628 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55bfe3e36be0 0x55bfe3e3ab20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.348+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 >> 172.21.15.40:0/3079161628 conn(0x55bfe3e2cda0 msgr2=0x55bfe3e39a30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.349+0000 7fa7beffd700 0 client.26063 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.349+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 shutdown_connections 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.349+0000 7fa7f9265b80 1 -- 172.21.15.40:0/3079161628 wait complete. 2023-07-14T11:13:11.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.349+0000 7fa7f9265b80 20 client.26063 trim_cache size 0 max 16384 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (1 inodes in the stack)", 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "88bd81f4-489e-47f0-a82d-3ec87899c170": { 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "88bd81f4-489e-47f0-a82d-3ec87899c170", 2023-07-14T11:13:11.359 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:13:11.360 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:13:11.360 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:13:11.360 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:13:11.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 msgr2=0x7fa7c80a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 0x7fa7c80a3890 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fa7d8009790 tx=0x7fa7d8009920 comp rx=0 tx=0).stop 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 msgr2=0x7fa7e0197f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fa7d000e9c0 tx=0x7fa7d000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 shutdown_connections 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa7c80a13e0 0x7fa7c80a3890 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa7e010c380 0x7fa7e019c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa7e0072850 0x7fa7e0197f70 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 --2- 172.21.15.40:0/2857529251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7e0073160 0x7fa7e01984b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:11.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 >> 172.21.15.40:0/2857529251 conn(0x7fa7e00fb5e0 msgr2=0x7fa7e010e430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:11.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.350+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 shutdown_connections 2023-07-14T11:13:11.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:11.351+0000 7fa7e59ff700 1 -- 172.21.15.40:0/2857529251 wait complete. 2023-07-14T11:13:11.375 INFO:tasks.cephfs.filesystem:scrub status for tag:88bd81f4-489e-47f0-a82d-3ec87899c170 - {'path': '/', 'tag': '88bd81f4-489e-47f0-a82d-3ec87899c170', 'options': 'recursive,force'} 2023-07-14T11:13:11.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: pgmap v34: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 29 MiB/s wr, 349 op/s 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub queued for path: / 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:11.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:11.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:11 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: pgmap v34: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 29 MiB/s wr, 349 op/s 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub queued for path: / 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:11 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: pgmap v34: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 36 KiB/s rd, 29 MiB/s wr, 349 op/s 2023-07-14T11:13:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub queued for path: / 2023-07-14T11:13:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:11 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:12.228 INFO:teuthology.orchestra.run.smithi040.stderr:13500000 of 50000000 tuples (27%) done (elapsed 46.72 s, remaining 126.32 s) 2023-07-14T11:13:12.377 INFO:teuthology.orchestra.run.smithi040.stderr:13600000 of 50000000 tuples (27%) done (elapsed 46.87 s, remaining 125.44 s) 2023-07-14T11:13:12.551 INFO:teuthology.orchestra.run.smithi040.stderr:13700000 of 50000000 tuples (27%) done (elapsed 47.04 s, remaining 124.65 s) 2023-07-14T11:13:13.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:13:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:12 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:13.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:13.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:13:13.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:12 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:13:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:12 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:13:13.143 INFO:teuthology.orchestra.run.smithi040.stderr:13800000 of 50000000 tuples (27%) done (elapsed 47.64 s, remaining 124.96 s) 2023-07-14T11:13:13.819 INFO:teuthology.orchestra.run.smithi040.stderr:13900000 of 50000000 tuples (27%) done (elapsed 48.31 s, remaining 125.47 s) 2023-07-14T11:13:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:13 smithi130 ceph-mon[129405]: pgmap v35: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 7.8 MiB/s wr, 306 op/s 2023-07-14T11:13:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:13 smithi040 ceph-mon[97954]: pgmap v35: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 7.8 MiB/s wr, 306 op/s 2023-07-14T11:13:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:13 smithi045 ceph-mon[128791]: pgmap v35: 129 pgs: 129 active+clean; 2.3 GiB data, 5.2 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 7.8 MiB/s wr, 306 op/s 2023-07-14T11:13:14.118 INFO:teuthology.orchestra.run.smithi040.stderr:14000000 of 50000000 tuples (28%) done (elapsed 48.61 s, remaining 125.00 s) 2023-07-14T11:13:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:14100000 of 50000000 tuples (28%) done (elapsed 48.78 s, remaining 124.19 s) 2023-07-14T11:13:14.527 INFO:teuthology.orchestra.run.smithi040.stderr:14200000 of 50000000 tuples (28%) done (elapsed 49.02 s, remaining 123.59 s) 2023-07-14T11:13:14.693 INFO:teuthology.orchestra.run.smithi040.stderr:14300000 of 50000000 tuples (28%) done (elapsed 49.19 s, remaining 122.79 s) 2023-07-14T11:13:14.886 INFO:teuthology.orchestra.run.smithi040.stderr:14400000 of 50000000 tuples (28%) done (elapsed 49.38 s, remaining 122.07 s) 2023-07-14T11:13:15.048 INFO:teuthology.orchestra.run.smithi040.stderr:14500000 of 50000000 tuples (29%) done (elapsed 49.54 s, remaining 121.29 s) 2023-07-14T11:13:15.262 INFO:teuthology.orchestra.run.smithi040.stderr:14600000 of 50000000 tuples (29%) done (elapsed 49.75 s, remaining 120.64 s) 2023-07-14T11:13:15.419 INFO:teuthology.orchestra.run.smithi040.stderr:14700000 of 50000000 tuples (29%) done (elapsed 49.91 s, remaining 119.86 s) 2023-07-14T11:13:15.589 INFO:teuthology.orchestra.run.smithi040.stderr:14800000 of 50000000 tuples (29%) done (elapsed 50.08 s, remaining 119.11 s) 2023-07-14T11:13:15.792 INFO:teuthology.orchestra.run.smithi040.stderr:14900000 of 50000000 tuples (29%) done (elapsed 50.28 s, remaining 118.46 s) 2023-07-14T11:13:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:15 smithi130 ceph-mon[129405]: pgmap v36: 129 pgs: 129 active+clean; 2.6 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 34 MiB/s wr, 396 op/s 2023-07-14T11:13:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:15 smithi040 ceph-mon[97954]: pgmap v36: 129 pgs: 129 active+clean; 2.6 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 34 MiB/s wr, 396 op/s 2023-07-14T11:13:15.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:15 smithi045 ceph-mon[128791]: pgmap v36: 129 pgs: 129 active+clean; 2.6 GiB data, 5.7 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 34 MiB/s wr, 396 op/s 2023-07-14T11:13:15.975 INFO:teuthology.orchestra.run.smithi040.stderr:15000000 of 50000000 tuples (30%) done (elapsed 50.47 s, remaining 117.76 s) 2023-07-14T11:13:16.143 INFO:teuthology.orchestra.run.smithi040.stderr:15100000 of 50000000 tuples (30%) done (elapsed 50.64 s, remaining 117.03 s) 2023-07-14T11:13:16.266 INFO:teuthology.orchestra.run.smithi040.stderr:15200000 of 50000000 tuples (30%) done (elapsed 50.76 s, remaining 116.21 s) 2023-07-14T11:13:16.451 INFO:teuthology.orchestra.run.smithi040.stderr:15300000 of 50000000 tuples (30%) done (elapsed 50.94 s, remaining 115.54 s) 2023-07-14T11:13:16.628 INFO:teuthology.orchestra.run.smithi040.stderr:15400000 of 50000000 tuples (30%) done (elapsed 51.12 s, remaining 114.86 s) 2023-07-14T11:13:16.777 INFO:teuthology.orchestra.run.smithi040.stderr:15500000 of 50000000 tuples (31%) done (elapsed 51.27 s, remaining 114.12 s) 2023-07-14T11:13:16.986 INFO:teuthology.orchestra.run.smithi040.stderr:15600000 of 50000000 tuples (31%) done (elapsed 51.48 s, remaining 113.52 s) 2023-07-14T11:13:17.195 INFO:teuthology.orchestra.run.smithi040.stderr:15700000 of 50000000 tuples (31%) done (elapsed 51.69 s, remaining 112.92 s) 2023-07-14T11:13:17.373 INFO:teuthology.orchestra.run.smithi040.stderr:15800000 of 50000000 tuples (31%) done (elapsed 51.87 s, remaining 112.27 s) 2023-07-14T11:13:17.559 INFO:teuthology.orchestra.run.smithi040.stderr:15900000 of 50000000 tuples (31%) done (elapsed 52.05 s, remaining 111.63 s) 2023-07-14T11:13:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:17 smithi130 ceph-mon[129405]: pgmap v37: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 52 MiB/s wr, 341 op/s 2023-07-14T11:13:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:13:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:17 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:17 smithi040 ceph-mon[97954]: pgmap v37: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 52 MiB/s wr, 341 op/s 2023-07-14T11:13:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:13:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:17 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:17 smithi045 ceph-mon[128791]: pgmap v37: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 35 KiB/s rd, 52 MiB/s wr, 341 op/s 2023-07-14T11:13:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:13:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:17 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:18.181 INFO:teuthology.orchestra.run.smithi040.stderr:16000000 of 50000000 tuples (32%) done (elapsed 52.67 s, remaining 111.93 s) 2023-07-14T11:13:18.470 INFO:teuthology.orchestra.run.smithi040.stderr:16100000 of 50000000 tuples (32%) done (elapsed 52.96 s, remaining 111.52 s) 2023-07-14T11:13:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:16200000 of 50000000 tuples (32%) done (elapsed 53.15 s, remaining 110.90 s) 2023-07-14T11:13:18.791 INFO:teuthology.orchestra.run.smithi040.stderr:16300000 of 50000000 tuples (32%) done (elapsed 53.28 s, remaining 110.16 s) 2023-07-14T11:13:18.975 INFO:teuthology.orchestra.run.smithi040.stderr:16400000 of 50000000 tuples (32%) done (elapsed 53.47 s, remaining 109.54 s) 2023-07-14T11:13:19.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:18 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:13:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:18 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:13:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:18 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:19.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:18 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:13:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:18 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:13:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:18 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:19.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:18 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:13:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:18 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:13:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:18 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:13:19.166 INFO:teuthology.orchestra.run.smithi040.stderr:16500000 of 50000000 tuples (33%) done (elapsed 53.66 s, remaining 108.94 s) 2023-07-14T11:13:19.304 INFO:teuthology.orchestra.run.smithi040.stderr:16600000 of 50000000 tuples (33%) done (elapsed 53.80 s, remaining 108.24 s) 2023-07-14T11:13:19.478 INFO:teuthology.orchestra.run.smithi040.stderr:16700000 of 50000000 tuples (33%) done (elapsed 53.97 s, remaining 107.62 s) 2023-07-14T11:13:19.689 INFO:teuthology.orchestra.run.smithi040.stderr:16800000 of 50000000 tuples (33%) done (elapsed 54.18 s, remaining 107.07 s) 2023-07-14T11:13:19.833 INFO:teuthology.orchestra.run.smithi040.stderr:16900000 of 50000000 tuples (33%) done (elapsed 54.33 s, remaining 106.40 s) 2023-07-14T11:13:19.995 INFO:teuthology.orchestra.run.smithi040.stderr:17000000 of 50000000 tuples (34%) done (elapsed 54.49 s, remaining 105.77 s) 2023-07-14T11:13:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:19 smithi130 ceph-mon[129405]: pgmap v38: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 51 MiB/s wr, 248 op/s 2023-07-14T11:13:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:19 smithi040 ceph-mon[97954]: pgmap v38: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 51 MiB/s wr, 248 op/s 2023-07-14T11:13:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:19 smithi045 ceph-mon[128791]: pgmap v38: 129 pgs: 129 active+clean; 2.9 GiB data, 6.1 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 51 MiB/s wr, 248 op/s 2023-07-14T11:13:20.173 INFO:teuthology.orchestra.run.smithi040.stderr:17100000 of 50000000 tuples (34%) done (elapsed 54.67 s, remaining 105.18 s) 2023-07-14T11:13:20.397 INFO:teuthology.orchestra.run.smithi040.stderr:17200000 of 50000000 tuples (34%) done (elapsed 54.89 s, remaining 104.67 s) 2023-07-14T11:13:20.582 INFO:teuthology.orchestra.run.smithi040.stderr:17300000 of 50000000 tuples (34%) done (elapsed 55.07 s, remaining 104.10 s) 2023-07-14T11:13:20.689 INFO:teuthology.orchestra.run.smithi040.stderr:17400000 of 50000000 tuples (34%) done (elapsed 55.18 s, remaining 103.39 s) 2023-07-14T11:13:20.865 INFO:teuthology.orchestra.run.smithi040.stderr:17500000 of 50000000 tuples (35%) done (elapsed 55.36 s, remaining 102.81 s) 2023-07-14T11:13:21.073 INFO:teuthology.orchestra.run.smithi040.stderr:17600000 of 50000000 tuples (35%) done (elapsed 55.57 s, remaining 102.29 s) 2023-07-14T11:13:21.355 INFO:teuthology.orchestra.run.smithi040.stderr:17700000 of 50000000 tuples (35%) done (elapsed 55.85 s, remaining 101.91 s) 2023-07-14T11:13:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:21 smithi130 ceph-mon[129405]: pgmap v39: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 72 MiB/s wr, 290 op/s 2023-07-14T11:13:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:21 smithi040 ceph-mon[97954]: pgmap v39: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 72 MiB/s wr, 290 op/s 2023-07-14T11:13:22.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:21 smithi045 ceph-mon[128791]: pgmap v39: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 72 MiB/s wr, 290 op/s 2023-07-14T11:13:22.415 INFO:teuthology.orchestra.run.smithi040.stderr:17800000 of 50000000 tuples (35%) done (elapsed 56.91 s, remaining 102.95 s) 2023-07-14T11:13:22.707 INFO:teuthology.orchestra.run.smithi040.stderr:17900000 of 50000000 tuples (35%) done (elapsed 57.20 s, remaining 102.58 s) 2023-07-14T11:13:22.953 INFO:teuthology.orchestra.run.smithi040.stderr:18000000 of 50000000 tuples (36%) done (elapsed 57.45 s, remaining 102.13 s) 2023-07-14T11:13:23.188 INFO:teuthology.orchestra.run.smithi040.stderr:18100000 of 50000000 tuples (36%) done (elapsed 57.68 s, remaining 101.66 s) 2023-07-14T11:13:23.321 INFO:teuthology.orchestra.run.smithi040.stderr:18200000 of 50000000 tuples (36%) done (elapsed 57.81 s, remaining 101.02 s) 2023-07-14T11:13:23.528 INFO:teuthology.orchestra.run.smithi040.stderr:18300000 of 50000000 tuples (36%) done (elapsed 58.02 s, remaining 100.51 s) 2023-07-14T11:13:23.718 INFO:teuthology.orchestra.run.smithi040.stderr:18400000 of 50000000 tuples (36%) done (elapsed 58.21 s, remaining 99.97 s) 2023-07-14T11:13:23.954 INFO:teuthology.orchestra.run.smithi040.stderr:18500000 of 50000000 tuples (37%) done (elapsed 58.45 s, remaining 99.52 s) 2023-07-14T11:13:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:23 smithi130 ceph-mon[129405]: pgmap v40: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 67 MiB/s wr, 150 op/s 2023-07-14T11:13:24.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:23 smithi040 ceph-mon[97954]: pgmap v40: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 67 MiB/s wr, 150 op/s 2023-07-14T11:13:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:23 smithi045 ceph-mon[128791]: pgmap v40: 129 pgs: 129 active+clean; 3.1 GiB data, 6.6 GiB used, 1.0 TiB / 1.0 TiB avail; 46 KiB/s rd, 67 MiB/s wr, 150 op/s 2023-07-14T11:13:24.097 INFO:teuthology.orchestra.run.smithi040.stderr:18600000 of 50000000 tuples (37%) done (elapsed 58.59 s, remaining 98.91 s) 2023-07-14T11:13:24.280 INFO:teuthology.orchestra.run.smithi040.stderr:18700000 of 50000000 tuples (37%) done (elapsed 58.77 s, remaining 98.37 s) 2023-07-14T11:13:24.481 INFO:teuthology.orchestra.run.smithi040.stderr:18800000 of 50000000 tuples (37%) done (elapsed 58.97 s, remaining 97.87 s) 2023-07-14T11:13:24.651 INFO:teuthology.orchestra.run.smithi040.stderr:18900000 of 50000000 tuples (37%) done (elapsed 59.14 s, remaining 97.32 s) 2023-07-14T11:13:24.802 INFO:teuthology.orchestra.run.smithi040.stderr:19000000 of 50000000 tuples (38%) done (elapsed 59.29 s, remaining 96.74 s) 2023-07-14T11:13:24.996 INFO:teuthology.orchestra.run.smithi040.stderr:19100000 of 50000000 tuples (38%) done (elapsed 59.49 s, remaining 96.24 s) 2023-07-14T11:13:25.223 INFO:teuthology.orchestra.run.smithi040.stderr:19200000 of 50000000 tuples (38%) done (elapsed 59.72 s, remaining 95.79 s) 2023-07-14T11:13:25.395 INFO:teuthology.orchestra.run.smithi040.stderr:19300000 of 50000000 tuples (38%) done (elapsed 59.89 s, remaining 95.26 s) 2023-07-14T11:13:25.546 INFO:teuthology.orchestra.run.smithi040.stderr:19400000 of 50000000 tuples (38%) done (elapsed 60.04 s, remaining 94.70 s) 2023-07-14T11:13:25.712 INFO:teuthology.orchestra.run.smithi040.stderr:19500000 of 50000000 tuples (39%) done (elapsed 60.20 s, remaining 94.17 s) 2023-07-14T11:13:25.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:25 smithi130 ceph-mon[129405]: pgmap v41: 129 pgs: 129 active+clean; 3.6 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 198 KiB/s rd, 107 MiB/s wr, 174 op/s 2023-07-14T11:13:25.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:25 smithi040 ceph-mon[97954]: pgmap v41: 129 pgs: 129 active+clean; 3.6 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 198 KiB/s rd, 107 MiB/s wr, 174 op/s 2023-07-14T11:13:25.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:25 smithi045 ceph-mon[128791]: pgmap v41: 129 pgs: 129 active+clean; 3.6 GiB data, 7.6 GiB used, 1.0 TiB / 1.0 TiB avail; 198 KiB/s rd, 107 MiB/s wr, 174 op/s 2023-07-14T11:13:25.956 INFO:teuthology.orchestra.run.smithi040.stderr:19600000 of 50000000 tuples (39%) done (elapsed 60.45 s, remaining 93.76 s) 2023-07-14T11:13:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:19700000 of 50000000 tuples (39%) done (elapsed 60.66 s, remaining 93.30 s) 2023-07-14T11:13:26.350 INFO:teuthology.orchestra.run.smithi040.stderr:19800000 of 50000000 tuples (39%) done (elapsed 60.84 s, remaining 92.80 s) 2023-07-14T11:13:26.467 INFO:teuthology.orchestra.run.smithi040.stderr:19900000 of 50000000 tuples (39%) done (elapsed 60.96 s, remaining 92.20 s) 2023-07-14T11:13:26.672 INFO:teuthology.orchestra.run.smithi040.stderr:20000000 of 50000000 tuples (40%) done (elapsed 61.17 s, remaining 91.75 s) 2023-07-14T11:13:26.844 INFO:teuthology.orchestra.run.smithi040.stderr:20100000 of 50000000 tuples (40%) done (elapsed 61.34 s, remaining 91.24 s) 2023-07-14T11:13:26.975 INFO:teuthology.orchestra.run.smithi040.stderr:20200000 of 50000000 tuples (40%) done (elapsed 61.47 s, remaining 90.68 s) 2023-07-14T11:13:27.170 INFO:teuthology.orchestra.run.smithi040.stderr:20300000 of 50000000 tuples (40%) done (elapsed 61.66 s, remaining 90.22 s) 2023-07-14T11:13:27.344 INFO:teuthology.orchestra.run.smithi040.stderr:20400000 of 50000000 tuples (40%) done (elapsed 61.84 s, remaining 89.72 s) 2023-07-14T11:13:27.481 INFO:teuthology.orchestra.run.smithi040.stderr:20500000 of 50000000 tuples (41%) done (elapsed 61.97 s, remaining 89.18 s) 2023-07-14T11:13:27.676 INFO:teuthology.orchestra.run.smithi040.stderr:20600000 of 50000000 tuples (41%) done (elapsed 62.17 s, remaining 88.73 s) 2023-07-14T11:13:27.859 INFO:teuthology.orchestra.run.smithi040.stderr:20700000 of 50000000 tuples (41%) done (elapsed 62.35 s, remaining 88.26 s) 2023-07-14T11:13:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:27 smithi130 ceph-mon[129405]: pgmap v42: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 102 MiB/s wr, 92 op/s 2023-07-14T11:13:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:27 smithi040 ceph-mon[97954]: pgmap v42: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 102 MiB/s wr, 92 op/s 2023-07-14T11:13:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:27 smithi045 ceph-mon[128791]: pgmap v42: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 205 KiB/s rd, 102 MiB/s wr, 92 op/s 2023-07-14T11:13:28.074 INFO:teuthology.orchestra.run.smithi040.stderr:20800000 of 50000000 tuples (41%) done (elapsed 62.57 s, remaining 87.83 s) 2023-07-14T11:13:28.285 INFO:teuthology.orchestra.run.smithi040.stderr:20900000 of 50000000 tuples (41%) done (elapsed 62.78 s, remaining 87.41 s) 2023-07-14T11:13:28.402 INFO:teuthology.orchestra.run.smithi040.stderr:21000000 of 50000000 tuples (42%) done (elapsed 62.89 s, remaining 86.85 s) 2023-07-14T11:13:28.598 INFO:teuthology.orchestra.run.smithi040.stderr:21100000 of 50000000 tuples (42%) done (elapsed 63.09 s, remaining 86.41 s) 2023-07-14T11:13:28.772 INFO:teuthology.orchestra.run.smithi040.stderr:21200000 of 50000000 tuples (42%) done (elapsed 63.26 s, remaining 85.94 s) 2023-07-14T11:13:28.910 INFO:teuthology.orchestra.run.smithi040.stderr:21300000 of 50000000 tuples (42%) done (elapsed 63.40 s, remaining 85.43 s) 2023-07-14T11:13:29.118 INFO:teuthology.orchestra.run.smithi040.stderr:21400000 of 50000000 tuples (42%) done (elapsed 63.61 s, remaining 85.01 s) 2023-07-14T11:13:29.275 INFO:teuthology.orchestra.run.smithi040.stderr:21500000 of 50000000 tuples (43%) done (elapsed 63.77 s, remaining 84.53 s) 2023-07-14T11:13:29.760 INFO:teuthology.orchestra.run.smithi040.stderr:21600000 of 50000000 tuples (43%) done (elapsed 64.25 s, remaining 84.48 s) 2023-07-14T11:13:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:29 smithi130 ceph-mon[129405]: pgmap v43: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 192 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:29 smithi040 ceph-mon[97954]: pgmap v43: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 192 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:29 smithi045 ceph-mon[128791]: pgmap v43: 129 pgs: 129 active+clean; 3.8 GiB data, 8.1 GiB used, 1.0 TiB / 1.0 TiB avail; 192 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:30.571 INFO:teuthology.orchestra.run.smithi040.stderr:21700000 of 50000000 tuples (43%) done (elapsed 65.06 s, remaining 84.85 s) 2023-07-14T11:13:30.877 INFO:teuthology.orchestra.run.smithi040.stderr:21800000 of 50000000 tuples (43%) done (elapsed 65.37 s, remaining 84.56 s) 2023-07-14T11:13:31.247 INFO:teuthology.orchestra.run.smithi040.stderr:21900000 of 50000000 tuples (43%) done (elapsed 65.74 s, remaining 84.35 s) 2023-07-14T11:13:31.459 INFO:teuthology.orchestra.run.smithi040.stderr:22000000 of 50000000 tuples (44%) done (elapsed 65.95 s, remaining 83.94 s) 2023-07-14T11:13:31.704 INFO:teuthology.orchestra.run.smithi040.stderr:22100000 of 50000000 tuples (44%) done (elapsed 66.20 s, remaining 83.57 s) 2023-07-14T11:13:31.901 INFO:teuthology.orchestra.run.smithi040.stderr:22200000 of 50000000 tuples (44%) done (elapsed 66.39 s, remaining 83.14 s) 2023-07-14T11:13:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:31 smithi130 ceph-mon[129405]: pgmap v44: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 219 KiB/s rd, 112 MiB/s wr, 91 op/s 2023-07-14T11:13:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:31 smithi040 ceph-mon[97954]: pgmap v44: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 219 KiB/s rd, 112 MiB/s wr, 91 op/s 2023-07-14T11:13:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:31 smithi045 ceph-mon[128791]: pgmap v44: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 219 KiB/s rd, 112 MiB/s wr, 91 op/s 2023-07-14T11:13:32.081 INFO:teuthology.orchestra.run.smithi040.stderr:22300000 of 50000000 tuples (44%) done (elapsed 66.57 s, remaining 82.70 s) 2023-07-14T11:13:32.189 INFO:teuthology.orchestra.run.smithi040.stderr:22400000 of 50000000 tuples (44%) done (elapsed 66.68 s, remaining 82.16 s) 2023-07-14T11:13:32.395 INFO:teuthology.orchestra.run.smithi040.stderr:22500000 of 50000000 tuples (45%) done (elapsed 66.89 s, remaining 81.75 s) 2023-07-14T11:13:32.554 INFO:teuthology.orchestra.run.smithi040.stderr:22600000 of 50000000 tuples (45%) done (elapsed 67.05 s, remaining 81.29 s) 2023-07-14T11:13:32.694 INFO:teuthology.orchestra.run.smithi040.stderr:22700000 of 50000000 tuples (45%) done (elapsed 67.19 s, remaining 80.80 s) 2023-07-14T11:13:32.885 INFO:teuthology.orchestra.run.smithi040.stderr:22800000 of 50000000 tuples (45%) done (elapsed 67.38 s, remaining 80.38 s) 2023-07-14T11:13:33.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:32 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:33.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:32 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:33.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:32 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:33.074 INFO:teuthology.orchestra.run.smithi040.stderr:22900000 of 50000000 tuples (45%) done (elapsed 67.57 s, remaining 79.96 s) 2023-07-14T11:13:33.224 INFO:teuthology.orchestra.run.smithi040.stderr:23000000 of 50000000 tuples (46%) done (elapsed 67.72 s, remaining 79.49 s) 2023-07-14T11:13:33.380 INFO:teuthology.orchestra.run.smithi040.stderr:23100000 of 50000000 tuples (46%) done (elapsed 67.87 s, remaining 79.04 s) 2023-07-14T11:13:33.589 INFO:teuthology.orchestra.run.smithi040.stderr:23200000 of 50000000 tuples (46%) done (elapsed 68.08 s, remaining 78.65 s) 2023-07-14T11:13:33.790 INFO:teuthology.orchestra.run.smithi040.stderr:23300000 of 50000000 tuples (46%) done (elapsed 68.28 s, remaining 78.25 s) 2023-07-14T11:13:33.973 INFO:teuthology.orchestra.run.smithi040.stderr:23400000 of 50000000 tuples (46%) done (elapsed 68.47 s, remaining 77.83 s) 2023-07-14T11:13:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:33 smithi130 ceph-mon[129405]: pgmap v45: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 187 KiB/s rd, 90 MiB/s wr, 49 op/s 2023-07-14T11:13:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:33 smithi040 ceph-mon[97954]: pgmap v45: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 187 KiB/s rd, 90 MiB/s wr, 49 op/s 2023-07-14T11:13:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:33 smithi045 ceph-mon[128791]: pgmap v45: 129 pgs: 129 active+clean; 4.1 GiB data, 8.6 GiB used, 1.0 TiB / 1.0 TiB avail; 187 KiB/s rd, 90 MiB/s wr, 49 op/s 2023-07-14T11:13:34.101 INFO:teuthology.orchestra.run.smithi040.stderr:23500000 of 50000000 tuples (47%) done (elapsed 68.59 s, remaining 77.35 s) 2023-07-14T11:13:34.295 INFO:teuthology.orchestra.run.smithi040.stderr:23600000 of 50000000 tuples (47%) done (elapsed 68.79 s, remaining 76.95 s) 2023-07-14T11:13:34.470 INFO:teuthology.orchestra.run.smithi040.stderr:23700000 of 50000000 tuples (47%) done (elapsed 68.96 s, remaining 76.53 s) 2023-07-14T11:13:34.618 INFO:teuthology.orchestra.run.smithi040.stderr:23800000 of 50000000 tuples (47%) done (elapsed 69.11 s, remaining 76.08 s) 2023-07-14T11:13:34.837 INFO:teuthology.orchestra.run.smithi040.stderr:23900000 of 50000000 tuples (47%) done (elapsed 69.33 s, remaining 75.71 s) 2023-07-14T11:13:35.015 INFO:teuthology.orchestra.run.smithi040.stderr:24000000 of 50000000 tuples (48%) done (elapsed 69.51 s, remaining 75.30 s) 2023-07-14T11:13:35.201 INFO:teuthology.orchestra.run.smithi040.stderr:24100000 of 50000000 tuples (48%) done (elapsed 69.69 s, remaining 74.90 s) 2023-07-14T11:13:35.359 INFO:teuthology.orchestra.run.smithi040.stderr:24200000 of 50000000 tuples (48%) done (elapsed 69.85 s, remaining 74.47 s) 2023-07-14T11:13:35.506 INFO:teuthology.orchestra.run.smithi040.stderr:24300000 of 50000000 tuples (48%) done (elapsed 70.00 s, remaining 74.03 s) 2023-07-14T11:13:35.696 INFO:teuthology.orchestra.run.smithi040.stderr:24400000 of 50000000 tuples (48%) done (elapsed 70.19 s, remaining 73.64 s) 2023-07-14T11:13:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:35 smithi130 ceph-mon[129405]: pgmap v46: 129 pgs: 129 active+clean; 4.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 231 KiB/s rd, 128 MiB/s wr, 64 op/s 2023-07-14T11:13:35.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:35 smithi040 ceph-mon[97954]: pgmap v46: 129 pgs: 129 active+clean; 4.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 231 KiB/s rd, 128 MiB/s wr, 64 op/s 2023-07-14T11:13:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:35 smithi045 ceph-mon[128791]: pgmap v46: 129 pgs: 129 active+clean; 4.5 GiB data, 9.5 GiB used, 1.0 TiB / 1.0 TiB avail; 231 KiB/s rd, 128 MiB/s wr, 64 op/s 2023-07-14T11:13:35.894 INFO:teuthology.orchestra.run.smithi040.stderr:24500000 of 50000000 tuples (49%) done (elapsed 70.39 s, remaining 73.26 s) 2023-07-14T11:13:36.061 INFO:teuthology.orchestra.run.smithi040.stderr:24600000 of 50000000 tuples (49%) done (elapsed 70.55 s, remaining 72.85 s) 2023-07-14T11:13:36.236 INFO:teuthology.orchestra.run.smithi040.stderr:24700000 of 50000000 tuples (49%) done (elapsed 70.73 s, remaining 72.45 s) 2023-07-14T11:13:36.423 INFO:teuthology.orchestra.run.smithi040.stderr:24800000 of 50000000 tuples (49%) done (elapsed 70.92 s, remaining 72.06 s) 2023-07-14T11:13:36.532 INFO:teuthology.orchestra.run.smithi040.stderr:24900000 of 50000000 tuples (49%) done (elapsed 71.02 s, remaining 71.60 s) 2023-07-14T11:13:36.699 INFO:teuthology.orchestra.run.smithi040.stderr:25000000 of 50000000 tuples (50%) done (elapsed 71.19 s, remaining 71.19 s) 2023-07-14T11:13:36.892 INFO:teuthology.orchestra.run.smithi040.stderr:25100000 of 50000000 tuples (50%) done (elapsed 71.38 s, remaining 70.82 s) 2023-07-14T11:13:37.336 INFO:teuthology.orchestra.run.smithi040.stderr:25200000 of 50000000 tuples (50%) done (elapsed 71.83 s, remaining 70.69 s) 2023-07-14T11:13:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:37 smithi130 ceph-mon[129405]: pgmap v47: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 142 KiB/s rd, 107 MiB/s wr, 59 op/s 2023-07-14T11:13:37.794 INFO:teuthology.orchestra.run.smithi040.stderr:25300000 of 50000000 tuples (50%) done (elapsed 72.29 s, remaining 70.57 s) 2023-07-14T11:13:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:37 smithi040 ceph-mon[97954]: pgmap v47: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 142 KiB/s rd, 107 MiB/s wr, 59 op/s 2023-07-14T11:13:37.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:37.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:37 smithi045 ceph-mon[128791]: pgmap v47: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 142 KiB/s rd, 107 MiB/s wr, 59 op/s 2023-07-14T11:13:37.979 INFO:teuthology.orchestra.run.smithi040.stderr:25400000 of 50000000 tuples (50%) done (elapsed 72.47 s, remaining 70.19 s) 2023-07-14T11:13:38.234 INFO:teuthology.orchestra.run.smithi040.stderr:25500000 of 50000000 tuples (51%) done (elapsed 72.73 s, remaining 69.87 s) 2023-07-14T11:13:38.779 INFO:teuthology.orchestra.run.smithi040.stderr:25600000 of 50000000 tuples (51%) done (elapsed 73.27 s, remaining 69.83 s) 2023-07-14T11:13:38.901 INFO:teuthology.orchestra.run.smithi040.stderr:25700000 of 50000000 tuples (51%) done (elapsed 73.39 s, remaining 69.40 s) 2023-07-14T11:13:39.734 INFO:teuthology.orchestra.run.smithi040.stderr:25800000 of 50000000 tuples (51%) done (elapsed 74.23 s, remaining 69.62 s) 2023-07-14T11:13:39.944 INFO:teuthology.orchestra.run.smithi040.stderr:25900000 of 50000000 tuples (51%) done (elapsed 74.44 s, remaining 69.26 s) 2023-07-14T11:13:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:39 smithi130 ceph-mon[129405]: pgmap v48: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 85 MiB/s wr, 51 op/s 2023-07-14T11:13:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:39 smithi040 ceph-mon[97954]: pgmap v48: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 85 MiB/s wr, 51 op/s 2023-07-14T11:13:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:39 smithi045 ceph-mon[128791]: pgmap v48: 129 pgs: 129 active+clean; 4.7 GiB data, 9.8 GiB used, 1.0 TiB / 1.0 TiB avail; 133 KiB/s rd, 85 MiB/s wr, 51 op/s 2023-07-14T11:13:40.180 INFO:teuthology.orchestra.run.smithi040.stderr:26000000 of 50000000 tuples (52%) done (elapsed 74.67 s, remaining 68.93 s) 2023-07-14T11:13:40.317 INFO:teuthology.orchestra.run.smithi040.stderr:26100000 of 50000000 tuples (52%) done (elapsed 74.81 s, remaining 68.50 s) 2023-07-14T11:13:40.502 INFO:teuthology.orchestra.run.smithi040.stderr:26200000 of 50000000 tuples (52%) done (elapsed 74.99 s, remaining 68.12 s) 2023-07-14T11:13:40.714 INFO:teuthology.orchestra.run.smithi040.stderr:26300000 of 50000000 tuples (52%) done (elapsed 75.21 s, remaining 67.77 s) 2023-07-14T11:13:40.884 INFO:teuthology.orchestra.run.smithi040.stderr:26400000 of 50000000 tuples (52%) done (elapsed 75.38 s, remaining 67.38 s) 2023-07-14T11:13:41.029 INFO:teuthology.orchestra.run.smithi040.stderr:26500000 of 50000000 tuples (53%) done (elapsed 75.52 s, remaining 66.97 s) 2023-07-14T11:13:41.230 INFO:teuthology.orchestra.run.smithi040.stderr:26600000 of 50000000 tuples (53%) done (elapsed 75.72 s, remaining 66.61 s) 2023-07-14T11:13:41.376 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:13:41.425 INFO:teuthology.orchestra.run.smithi040.stderr:26700000 of 50000000 tuples (53%) done (elapsed 75.92 s, remaining 66.25 s) 2023-07-14T11:13:41.602 INFO:teuthology.orchestra.run.smithi040.stderr:26800000 of 50000000 tuples (53%) done (elapsed 76.09 s, remaining 65.87 s) 2023-07-14T11:13:41.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 -- 172.21.15.40:0/1735942988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd80ffae0 msgr2=0x7f9bd80fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 --2- 172.21.15.40:0/1735942988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd80ffae0 0x7f9bd80fff50 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7f9bc80098f0 tx=0x7f9bc8009c00 comp rx=0 tx=0).stop 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 -- 172.21.15.40:0/1735942988 shutdown_connections 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 --2- 172.21.15.40:0/1735942988 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd8100490 0x7f9bd810c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 --2- 172.21.15.40:0/1735942988 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8105af0 0x7f9bd8105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 --2- 172.21.15.40:0/1735942988 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd80ffae0 0x7f9bd80fff50 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 -- 172.21.15.40:0/1735942988 >> 172.21.15.40:0/1735942988 conn(0x7f9bd80fb5e0 msgr2=0x7f9bd80fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 -- 172.21.15.40:0/1735942988 shutdown_connections 2023-07-14T11:13:41.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.606+0000 7f9be05ff700 1 -- 172.21.15.40:0/1735942988 wait complete. 2023-07-14T11:13:41.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.607+0000 7f9be05ff700 1 Processor -- start 2023-07-14T11:13:41.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.607+0000 7f9be05ff700 1 -- start start 2023-07-14T11:13:41.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.607+0000 7f9be05ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 0x7f9bd8197e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.607+0000 7f9be05ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9be05ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd8105af0 0x7f9bd819c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9be05ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819ce30 con 0x7f9bd80ffae0 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9be05ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819cfa0 con 0x7f9bd8105af0 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9be05ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9bd819d160 con 0x7f9bd8100490 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 0x7f9bd8197e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59332/0 (socket says 172.21.15.40:59332) 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 -- 172.21.15.40:0/3062030075 learned_addr learned my addr 172.21.15.40:0/3062030075 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd8105af0 0x7f9bd819c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 -- 172.21.15.40:0/3062030075 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd8105af0 msgr2=0x7f9bd819c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd8105af0 0x7f9bd819c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 -- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 msgr2=0x7f9bd8197e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 0x7f9bd8197e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bddb9a700 1 -- 172.21.15.40:0/3062030075 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9bd40095f0 con 0x7f9bd8100490 2023-07-14T11:13:41.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.608+0000 7f9bde39b700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 0x7f9bd8197e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:41.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f9bc8009810 tx=0x7f9bc800fa00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9bcf7fe700 1 -- 172.21.15.40:0/3062030075 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bc801c070 con 0x7f9bd8100490 2023-07-14T11:13:41.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9bcf7fe700 1 -- 172.21.15.40:0/3062030075 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bc800bb20 con 0x7f9bd8100490 2023-07-14T11:13:41.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9bc8009520 con 0x7f9bd8100490 2023-07-14T11:13:41.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9bcf7fe700 1 -- 172.21.15.40:0/3062030075 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bc8004930 con 0x7f9bd8100490 2023-07-14T11:13:41.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.609+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bd819d670 con 0x7f9bd8100490 2023-07-14T11:13:41.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.614+0000 7f9bcf7fe700 1 -- 172.21.15.40:0/3062030075 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f9bc8020070 con 0x7f9bd8100490 2023-07-14T11:13:41.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.614+0000 7f9bf3e7eb80 1 Processor -- start 2023-07-14T11:13:41.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 -- start start 2023-07-14T11:13:41.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68dea6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df7eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68cf0cb0 con 0x558b68dea2d0 2023-07-14T11:13:41.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68d43a40 con 0x558b68deabe0 2023-07-14T11:13:41.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.615+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68d3ced0 con 0x558b68df5ad0 2023-07-14T11:13:41.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48936/0 (socket says 172.21.15.40:48936) 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 -- 172.21.15.40:0/2697096125 learned_addr learned my addr 172.21.15.40:0/2697096125 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df7eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68dea6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 -- 172.21.15.40:0/2697096125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 msgr2=0x558b68df7eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df7eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 -- 172.21.15.40:0/2697096125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 msgr2=0x558b68dea6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68dea6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 -- 172.21.15.40:0/2697096125 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558b68b6ade0 con 0x558b68deabe0 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bcf7fe700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 0x7f9bc40a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.616+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f9bd4011780 tx=0x7f9bd4011a90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d7d677aa233e44ac server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.617+0000 7f9bcf7fe700 1 -- 172.21.15.40:0/3062030075 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9bc80cdcc0 con 0x7f9bd8100490 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.617+0000 7f9bccbfe700 1 -- 172.21.15.40:0/2697096125 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd4004020 con 0x558b68deabe0 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.617+0000 7f9bccbfe700 1 -- 172.21.15.40:0/2697096125 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bd4004830 con 0x558b68deabe0 2023-07-14T11:13:41.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.617+0000 7f9bccbfe700 1 -- 172.21.15.40:0/2697096125 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd4005410 con 0x558b68deabe0 2023-07-14T11:13:41.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.617+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 0x7f9bc40a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2697096125 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 msgr2=0x558b68df5590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f9bd4011780 tx=0x7f9bd4011a90 comp rx=0 tx=0).stop 2023-07-14T11:13:41.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 0x7f9bc40a3770 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0007500 tx=0x7f9bd000b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2697096125 shutdown_connections 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df7eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68dea6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2697096125 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df5590 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2697096125 >> 172.21.15.40:0/2697096125 conn(0x558b68de0da0 msgr2=0x558b68de66a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2697096125 shutdown_connections 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.618+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2697096125 wait complete. 2023-07-14T11:13:41.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.619+0000 7f9bf3e7eb80 1 -- start start 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.619+0000 7f9bf3e7eb80 1 Processor -- start 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 0x558b68df2e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68cdcac0 con 0x558b68df5ad0 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68cdc7c0 con 0x558b68deabe0 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68bc0920 con 0x558b68dea2d0 2023-07-14T11:13:41.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bf3e7eb80 10 client.0 _ll_register_callbacks cb 0x558b68cda760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bddb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59336/0 (socket says 172.21.15.40:59336) 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bddb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48938/0 (socket says 172.21.15.40:48938) 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bde39b700 1 -- 172.21.15.40:0/2121646587 learned_addr learned my addr 172.21.15.40:0/2121646587 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 0x558b68df2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.620+0000 7f9bde39b700 1 -- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 msgr2=0x558b68df27a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bde39b700 1 -- 172.21.15.40:0/2121646587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 msgr2=0x558b68df2e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 0x558b68df2e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bde39b700 1 -- 172.21.15.40:0/2121646587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9bd4011440 con 0x558b68dea2d0 2023-07-14T11:13:41.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:41.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 0x558b68df2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:41.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bde39b700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f9bd4011750 tx=0x7f9bd4005330 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b0d86d3d5e78cede server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bc1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:41.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd4005580 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558b68c964f0 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9bd4004030 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=32) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9bd40056e0 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bf3e7eb80 10 client.26053 fetch_fsmap learned FSMap version 32 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.621+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=32,monmap=4+}) v3 -- 0x558b68b53c60 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x558b68c964f0 con 0x558b68dea2d0 2023-07-14T11:13:41.630 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 32) v1 ==== 7494+0+0 (secure 0 0 0) 0x7f9bd400f5a0 con 0x558b68dea2d0 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 10 client.26053 fetch_fsmap finished waiting for FSMap version 32 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 10 client.26053 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 20 client.26053 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e05350 0x558b68e07760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 4 client.26053 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558b68d35eb0 con 0x558b68e05350 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.622+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e05350 0x558b68e07760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.631 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.623+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e05350 0x558b68e07760 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d580438359a33a00 server_cookie=79824e71cdd0c7d6 in_seq=0 out_seq=0 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.623+0000 7f9bc1ffb700 10 client.26053 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/2121646587 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x558b68d35eb0 con 0x558b68e05350 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bc1ffb700 10 client.26053 handle_command_reply: tid=0 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 client.26053 shutdown 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 msgr2=0x558b68df54b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f9bd4011750 tx=0x7f9bd4005330 comp rx=0 tx=0).stop 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 shutdown_connections 2023-07-14T11:13:41.632 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e05350 0x558b68e07760 crc :-1 s=READY pgs=71 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68df5ad0 0x558b68df2e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68dea2d0 0x558b68df54b0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bc1ffb700 0 client.26053 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/2121646587 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68df27a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 >> 172.21.15.40:0/2121646587 conn(0x558b68de0da0 msgr2=0x558b68de6290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 shutdown_connections 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/2121646587 wait complete. 2023-07-14T11:13:41.633 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.624+0000 7f9bf3e7eb80 20 client.26053 trim_cache size 0 max 16384 2023-07-14T11:13:41.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.644+0000 7f9bf3e7eb80 1 Processor -- start 2023-07-14T11:13:41.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.644+0000 7f9bf3e7eb80 1 -- start start 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 0x558b68e01740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68dddfa0 con 0x558b68deabe0 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68df3af0 con 0x558b68dea2d0 2023-07-14T11:13:41.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68de0970 con 0x558b68df5ad0 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59352/0 (socket says 172.21.15.40:59352) 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48954/0 (socket says 172.21.15.40:48954) 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/1427841133 learned_addr learned my addr 172.21.15.40:0/1427841133 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:41.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 0x558b68e01740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/1427841133 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 msgr2=0x558b68dfc4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/1427841133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 msgr2=0x558b68e01740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 0x558b68e01740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/1427841133 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558b68b91250 con 0x558b68df5ad0 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.645+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 0x558b68e01740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.646+0000 7f9bde39b700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.646+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0009e90 tx=0x7f9bd0016000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=81d992f2df18e255 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.646+0000 7f9bcd3ff700 1 -- 172.21.15.40:0/1427841133 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd000e040 con 0x558b68df5ad0 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.646+0000 7f9bcd3ff700 1 -- 172.21.15.40:0/1427841133 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bd000f040 con 0x558b68df5ad0 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.646+0000 7f9bcd3ff700 1 -- 172.21.15.40:0/1427841133 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd00157b0 con 0x558b68df5ad0 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/1427841133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 msgr2=0x558b68df5050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0009e90 tx=0x7f9bd0016000 comp rx=0 tx=0).stop 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/1427841133 shutdown_connections 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68df5050 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68dea2d0 0x558b68dfc4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/1427841133 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68deabe0 0x558b68e01740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/1427841133 >> 172.21.15.40:0/1427841133 conn(0x558b68de0da0 msgr2=0x558b68e0c080 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/1427841133 shutdown_connections 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/1427841133 wait complete. 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.647+0000 7f9bf3e7eb80 1 -- start start 2023-07-14T11:13:41.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.648+0000 7f9bf3e7eb80 1 Processor -- start 2023-07-14T11:13:41.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.648+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68e4d350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68e4d890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68e518a0 con 0x558b68dea2d0 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68e51a10 con 0x558b68deabe0 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558b68e51bd0 con 0x558b68df5ad0 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bf3e7eb80 10 client.0 _ll_register_callbacks cb 0x7f9bc80d1640 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:13:41.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bde39b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68e4d350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59358/0 (socket says 172.21.15.40:59358) 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/3935426234 learned_addr learned my addr 172.21.15.40:0/3935426234 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bddb9a700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68e4d890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 msgr2=0x558b68e4d890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68e4d890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/3935426234 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 msgr2=0x558b68e4d350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68e4d350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.649+0000 7f9bdeb9c700 1 -- 172.21.15.40:0/3935426234 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558b68b53c60 con 0x558b68df5ad0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bdeb9c700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0008010 tx=0x7f9bd0024b10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=ee3efe2e9a4b0b75 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bc1ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9bd000e050 con 0x558b68df5ad0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f9bd4004830 con 0x558b68df5ad0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9bd0015840 con 0x558b68df5ad0 2023-07-14T11:13:41.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=32) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9bd0016e70 con 0x558b68df5ad0 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bf3e7eb80 10 client.26059 fetch_fsmap learned FSMap version 32 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=32,monmap=4+}) v3 -- 0x7f9bd0007c70 con 0x558b68df5ad0 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.650+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9bd4004830 con 0x558b68df5ad0 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 32) v1 ==== 7494+0+0 (secure 0 0 0) 0x7f9bd0036a70 con 0x558b68df5ad0 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 10 client.26059 fetch_fsmap finished waiting for FSMap version 32 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 10 client.26059 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 20 client.26059 populate_metadata read hostname 'smithi040' 2023-07-14T11:13:41.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e5a520 0x558b68e5c930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 4 client.26059 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x558b68cf8050 con 0x558b68e5a520 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bde39b700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e5a520 0x558b68e5c930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bde39b700 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e5a520 0x558b68e5c930 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3301843785195629 server_cookie=6d425df68b460512 in_seq=0 out_seq=0 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.651+0000 7f9bc1ffb700 10 client.26059 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bc1ffb700 1 -- 172.21.15.40:0/3935426234 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x558b68cf8050 con 0x558b68e5a520 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bc1ffb700 10 client.26059 handle_command_reply: tid=0 2023-07-14T11:13:41.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 client.26059 shutdown 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 msgr2=0x558b68e512f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0008010 tx=0x7f9bd0024b10 comp rx=0 tx=0).stop 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 shutdown_connections 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558b68e5a520 0x558b68e5c930 crc :-1 s=READY pgs=72 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558b68df5ad0 0x558b68e512f0 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558b68dea2d0 0x558b68e4d350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 --2- 172.21.15.40:0/3935426234 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558b68deabe0 0x558b68e4d890 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 >> 172.21.15.40:0/3935426234 conn(0x558b68de0da0 msgr2=0x558b68e01740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bc1ffb700 0 client.26059 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 shutdown_connections 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 1 -- 172.21.15.40:0/3935426234 wait complete. 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.652+0000 7f9bf3e7eb80 20 client.26059 trim_cache size 0 max 16384 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:13:41.662 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (1 inodes in the stack)", 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: "88bd81f4-489e-47f0-a82d-3ec87899c170": { 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "88bd81f4-489e-47f0-a82d-3ec87899c170", 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:13:41.663 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:13:41.664 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:13:41.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 msgr2=0x7f9bc40a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 0x7f9bc40a3770 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f9bd0007500 tx=0x7f9bd000b040 comp rx=0 tx=0).stop 2023-07-14T11:13:41.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 msgr2=0x7f9bd8198390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f9bc8009810 tx=0x7f9bc800fa00 comp rx=0 tx=0).stop 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 shutdown_connections 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9bc40a12c0 0x7f9bc40a3770 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9bd8105af0 0x7f9bd819c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9bd80ffae0 0x7f9bd8197e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 --2- 172.21.15.40:0/3062030075 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9bd8100490 0x7f9bd8198390 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:13:41.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 >> 172.21.15.40:0/3062030075 conn(0x7f9bd80fb5e0 msgr2=0x7f9bd810ae80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:13:41.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 shutdown_connections 2023-07-14T11:13:41.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:13:41.653+0000 7f9be05ff700 1 -- 172.21.15.40:0/3062030075 wait complete. 2023-07-14T11:13:41.674 INFO:tasks.cephfs.filesystem:scrub status for tag:88bd81f4-489e-47f0-a82d-3ec87899c170 - {'path': '/', 'tag': '88bd81f4-489e-47f0-a82d-3ec87899c170', 'options': 'recursive,force'} 2023-07-14T11:13:41.738 INFO:teuthology.orchestra.run.smithi040.stderr:26900000 of 50000000 tuples (53%) done (elapsed 76.23 s, remaining 65.46 s) 2023-07-14T11:13:41.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:41 smithi040 ceph-mon[97954]: pgmap v49: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 144 KiB/s rd, 148 MiB/s wr, 104 op/s 2023-07-14T11:13:41.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:41 smithi040 ceph-mon[97954]: mgrmap e24: x(active, since 92s), standbys: z, y 2023-07-14T11:13:41.900 INFO:teuthology.orchestra.run.smithi040.stderr:27000000 of 50000000 tuples (54%) done (elapsed 76.39 s, remaining 65.08 s) 2023-07-14T11:13:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:41 smithi130 ceph-mon[129405]: pgmap v49: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 144 KiB/s rd, 148 MiB/s wr, 104 op/s 2023-07-14T11:13:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:41 smithi130 ceph-mon[129405]: mgrmap e24: x(active, since 92s), standbys: z, y 2023-07-14T11:13:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:41 smithi045 ceph-mon[128791]: pgmap v49: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 144 KiB/s rd, 148 MiB/s wr, 104 op/s 2023-07-14T11:13:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:41 smithi045 ceph-mon[128791]: mgrmap e24: x(active, since 92s), standbys: z, y 2023-07-14T11:13:42.116 INFO:teuthology.orchestra.run.smithi040.stderr:27100000 of 50000000 tuples (54%) done (elapsed 76.61 s, remaining 64.74 s) 2023-07-14T11:13:42.358 INFO:teuthology.orchestra.run.smithi040.stderr:27200000 of 50000000 tuples (54%) done (elapsed 76.85 s, remaining 64.42 s) 2023-07-14T11:13:43.016 INFO:teuthology.orchestra.run.smithi040.stderr:27300000 of 50000000 tuples (54%) done (elapsed 77.51 s, remaining 64.45 s) 2023-07-14T11:13:43.122 INFO:teuthology.orchestra.run.smithi040.stderr:27400000 of 50000000 tuples (54%) done (elapsed 77.62 s, remaining 64.02 s) 2023-07-14T11:13:43.469 INFO:teuthology.orchestra.run.smithi040.stderr:27500000 of 50000000 tuples (55%) done (elapsed 77.96 s, remaining 63.79 s) 2023-07-14T11:13:44.016 INFO:teuthology.orchestra.run.smithi040.stderr:27600000 of 50000000 tuples (55%) done (elapsed 78.51 s, remaining 63.72 s) 2023-07-14T11:13:44.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:43 smithi130 ceph-mon[129405]: pgmap v50: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 120 MiB/s wr, 87 op/s 2023-07-14T11:13:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:43 smithi040 ceph-mon[97954]: pgmap v50: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 120 MiB/s wr, 87 op/s 2023-07-14T11:13:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:43 smithi045 ceph-mon[128791]: pgmap v50: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 117 KiB/s rd, 120 MiB/s wr, 87 op/s 2023-07-14T11:13:44.233 INFO:teuthology.orchestra.run.smithi040.stderr:27700000 of 50000000 tuples (55%) done (elapsed 78.73 s, remaining 63.38 s) 2023-07-14T11:13:44.438 INFO:teuthology.orchestra.run.smithi040.stderr:27800000 of 50000000 tuples (55%) done (elapsed 78.93 s, remaining 63.03 s) 2023-07-14T11:13:44.615 INFO:teuthology.orchestra.run.smithi040.stderr:27900000 of 50000000 tuples (55%) done (elapsed 79.11 s, remaining 62.66 s) 2023-07-14T11:13:44.766 INFO:teuthology.orchestra.run.smithi040.stderr:28000000 of 50000000 tuples (56%) done (elapsed 79.26 s, remaining 62.27 s) 2023-07-14T11:13:44.931 INFO:teuthology.orchestra.run.smithi040.stderr:28100000 of 50000000 tuples (56%) done (elapsed 79.42 s, remaining 61.90 s) 2023-07-14T11:13:45.130 INFO:teuthology.orchestra.run.smithi040.stderr:28200000 of 50000000 tuples (56%) done (elapsed 79.62 s, remaining 61.55 s) 2023-07-14T11:13:45.327 INFO:teuthology.orchestra.run.smithi040.stderr:28300000 of 50000000 tuples (56%) done (elapsed 79.82 s, remaining 61.20 s) 2023-07-14T11:13:45.489 INFO:teuthology.orchestra.run.smithi040.stderr:28400000 of 50000000 tuples (56%) done (elapsed 79.98 s, remaining 60.83 s) 2023-07-14T11:13:45.614 INFO:teuthology.orchestra.run.smithi040.stderr:28500000 of 50000000 tuples (57%) done (elapsed 80.11 s, remaining 60.43 s) 2023-07-14T11:13:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:45 smithi130 ceph-mon[129405]: pgmap v51: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 184 KiB/s rd, 139 MiB/s wr, 165 op/s 2023-07-14T11:13:45.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:45 smithi040 ceph-mon[97954]: pgmap v51: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 184 KiB/s rd, 139 MiB/s wr, 165 op/s 2023-07-14T11:13:45.806 INFO:teuthology.orchestra.run.smithi040.stderr:28600000 of 50000000 tuples (57%) done (elapsed 80.30 s, remaining 60.08 s) 2023-07-14T11:13:45.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:45 smithi045 ceph-mon[128791]: pgmap v51: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 184 KiB/s rd, 139 MiB/s wr, 165 op/s 2023-07-14T11:13:45.979 INFO:teuthology.orchestra.run.smithi040.stderr:28700000 of 50000000 tuples (57%) done (elapsed 80.47 s, remaining 59.72 s) 2023-07-14T11:13:46.117 INFO:teuthology.orchestra.run.smithi040.stderr:28800000 of 50000000 tuples (57%) done (elapsed 80.61 s, remaining 59.34 s) 2023-07-14T11:13:46.295 INFO:teuthology.orchestra.run.smithi040.stderr:28900000 of 50000000 tuples (57%) done (elapsed 80.79 s, remaining 58.98 s) 2023-07-14T11:13:46.496 INFO:teuthology.orchestra.run.smithi040.stderr:29000000 of 50000000 tuples (58%) done (elapsed 80.99 s, remaining 58.65 s) 2023-07-14T11:13:46.649 INFO:teuthology.orchestra.run.smithi040.stderr:29100000 of 50000000 tuples (58%) done (elapsed 81.14 s, remaining 58.28 s) 2023-07-14T11:13:46.830 INFO:teuthology.orchestra.run.smithi040.stderr:29200000 of 50000000 tuples (58%) done (elapsed 81.32 s, remaining 57.93 s) 2023-07-14T11:13:47.188 INFO:teuthology.orchestra.run.smithi040.stderr:29300000 of 50000000 tuples (58%) done (elapsed 81.68 s, remaining 57.71 s) 2023-07-14T11:13:47.757 INFO:teuthology.orchestra.run.smithi040.stderr:29400000 of 50000000 tuples (58%) done (elapsed 82.25 s, remaining 57.63 s) 2023-07-14T11:13:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:47 smithi130 ceph-mon[129405]: pgmap v52: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 101 MiB/s wr, 150 op/s 2023-07-14T11:13:47.811 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:47.811 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:47 smithi040 ceph-mon[97954]: pgmap v52: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 101 MiB/s wr, 150 op/s 2023-07-14T11:13:47.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:13:47.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:47 smithi045 ceph-mon[128791]: pgmap v52: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 141 KiB/s rd, 101 MiB/s wr, 150 op/s 2023-07-14T11:13:48.096 INFO:teuthology.orchestra.run.smithi040.stderr:29500000 of 50000000 tuples (59%) done (elapsed 82.59 s, remaining 57.39 s) 2023-07-14T11:13:48.489 INFO:teuthology.orchestra.run.smithi040.stderr:29600000 of 50000000 tuples (59%) done (elapsed 82.98 s, remaining 57.19 s) 2023-07-14T11:13:48.712 INFO:teuthology.orchestra.run.smithi040.stderr:29700000 of 50000000 tuples (59%) done (elapsed 83.20 s, remaining 56.87 s) 2023-07-14T11:13:48.926 INFO:teuthology.orchestra.run.smithi040.stderr:29800000 of 50000000 tuples (59%) done (elapsed 83.42 s, remaining 56.55 s) 2023-07-14T11:13:49.066 INFO:teuthology.orchestra.run.smithi040.stderr:29900000 of 50000000 tuples (59%) done (elapsed 83.56 s, remaining 56.17 s) 2023-07-14T11:13:49.264 INFO:teuthology.orchestra.run.smithi040.stderr:30000000 of 50000000 tuples (60%) done (elapsed 83.76 s, remaining 55.84 s) 2023-07-14T11:13:49.448 INFO:teuthology.orchestra.run.smithi040.stderr:30100000 of 50000000 tuples (60%) done (elapsed 83.94 s, remaining 55.50 s) 2023-07-14T11:13:49.590 INFO:teuthology.orchestra.run.smithi040.stderr:30200000 of 50000000 tuples (60%) done (elapsed 84.08 s, remaining 55.13 s) 2023-07-14T11:13:49.786 INFO:teuthology.orchestra.run.smithi040.stderr:30300000 of 50000000 tuples (60%) done (elapsed 84.28 s, remaining 54.79 s) 2023-07-14T11:13:49.965 INFO:teuthology.orchestra.run.smithi040.stderr:30400000 of 50000000 tuples (60%) done (elapsed 84.46 s, remaining 54.45 s) 2023-07-14T11:13:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:49 smithi130 ceph-mon[129405]: pgmap v53: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 82 MiB/s wr, 131 op/s 2023-07-14T11:13:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:49 smithi040 ceph-mon[97954]: pgmap v53: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 82 MiB/s wr, 131 op/s 2023-07-14T11:13:50.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:49 smithi045 ceph-mon[128791]: pgmap v53: 129 pgs: 129 active+clean; 5.3 GiB data, 11 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 82 MiB/s wr, 131 op/s 2023-07-14T11:13:50.110 INFO:teuthology.orchestra.run.smithi040.stderr:30500000 of 50000000 tuples (61%) done (elapsed 84.60 s, remaining 54.09 s) 2023-07-14T11:13:50.339 INFO:teuthology.orchestra.run.smithi040.stderr:30600000 of 50000000 tuples (61%) done (elapsed 84.83 s, remaining 53.78 s) 2023-07-14T11:13:50.487 INFO:teuthology.orchestra.run.smithi040.stderr:30700000 of 50000000 tuples (61%) done (elapsed 84.98 s, remaining 53.42 s) 2023-07-14T11:13:50.685 INFO:teuthology.orchestra.run.smithi040.stderr:30800000 of 50000000 tuples (61%) done (elapsed 85.18 s, remaining 53.10 s) 2023-07-14T11:13:50.876 INFO:teuthology.orchestra.run.smithi040.stderr:30900000 of 50000000 tuples (61%) done (elapsed 85.37 s, remaining 52.77 s) 2023-07-14T11:13:50.988 INFO:teuthology.orchestra.run.smithi040.stderr:31000000 of 50000000 tuples (62%) done (elapsed 85.48 s, remaining 52.39 s) 2023-07-14T11:13:51.139 INFO:teuthology.orchestra.run.smithi040.stderr:31100000 of 50000000 tuples (62%) done (elapsed 85.63 s, remaining 52.04 s) 2023-07-14T11:13:51.311 INFO:teuthology.orchestra.run.smithi040.stderr:31200000 of 50000000 tuples (62%) done (elapsed 85.80 s, remaining 51.70 s) 2023-07-14T11:13:51.442 INFO:teuthology.orchestra.run.smithi040.stderr:31300000 of 50000000 tuples (62%) done (elapsed 85.93 s, remaining 51.34 s) 2023-07-14T11:13:51.745 INFO:teuthology.orchestra.run.smithi040.stderr:31400000 of 50000000 tuples (62%) done (elapsed 86.24 s, remaining 51.08 s) 2023-07-14T11:13:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:51 smithi130 ceph-mon[129405]: pgmap v54: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 143 MiB/s wr, 168 op/s 2023-07-14T11:13:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:51 smithi040 ceph-mon[97954]: pgmap v54: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 143 MiB/s wr, 168 op/s 2023-07-14T11:13:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:51 smithi045 ceph-mon[128791]: pgmap v54: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 111 KiB/s rd, 143 MiB/s wr, 168 op/s 2023-07-14T11:13:52.212 INFO:teuthology.orchestra.run.smithi040.stderr:31500000 of 50000000 tuples (63%) done (elapsed 86.71 s, remaining 50.92 s) 2023-07-14T11:13:52.514 INFO:teuthology.orchestra.run.smithi040.stderr:31600000 of 50000000 tuples (63%) done (elapsed 87.01 s, remaining 50.66 s) 2023-07-14T11:13:52.695 INFO:teuthology.orchestra.run.smithi040.stderr:31700000 of 50000000 tuples (63%) done (elapsed 87.19 s, remaining 50.33 s) 2023-07-14T11:13:52.866 INFO:teuthology.orchestra.run.smithi040.stderr:31800000 of 50000000 tuples (63%) done (elapsed 87.36 s, remaining 50.00 s) 2023-07-14T11:13:53.019 INFO:teuthology.orchestra.run.smithi040.stderr:31900000 of 50000000 tuples (63%) done (elapsed 87.51 s, remaining 49.65 s) 2023-07-14T11:13:53.189 INFO:teuthology.orchestra.run.smithi040.stderr:32000000 of 50000000 tuples (64%) done (elapsed 87.68 s, remaining 49.32 s) 2023-07-14T11:13:53.413 INFO:teuthology.orchestra.run.smithi040.stderr:32100000 of 50000000 tuples (64%) done (elapsed 87.91 s, remaining 49.02 s) 2023-07-14T11:13:53.610 INFO:teuthology.orchestra.run.smithi040.stderr:32200000 of 50000000 tuples (64%) done (elapsed 88.10 s, remaining 48.70 s) 2023-07-14T11:13:53.789 INFO:teuthology.orchestra.run.smithi040.stderr:32300000 of 50000000 tuples (64%) done (elapsed 88.28 s, remaining 48.38 s) 2023-07-14T11:13:53.919 INFO:teuthology.orchestra.run.smithi040.stderr:32400000 of 50000000 tuples (64%) done (elapsed 88.41 s, remaining 48.03 s) 2023-07-14T11:13:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:53 smithi130 ceph-mon[129405]: pgmap v55: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 80 MiB/s wr, 114 op/s 2023-07-14T11:13:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:53 smithi040 ceph-mon[97954]: pgmap v55: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 80 MiB/s wr, 114 op/s 2023-07-14T11:13:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:53 smithi045 ceph-mon[128791]: pgmap v55: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 100 KiB/s rd, 80 MiB/s wr, 114 op/s 2023-07-14T11:13:54.097 INFO:teuthology.orchestra.run.smithi040.stderr:32500000 of 50000000 tuples (65%) done (elapsed 88.59 s, remaining 47.70 s) 2023-07-14T11:13:54.262 INFO:teuthology.orchestra.run.smithi040.stderr:32600000 of 50000000 tuples (65%) done (elapsed 88.75 s, remaining 47.37 s) 2023-07-14T11:13:54.383 INFO:teuthology.orchestra.run.smithi040.stderr:32700000 of 50000000 tuples (65%) done (elapsed 88.88 s, remaining 47.02 s) 2023-07-14T11:13:54.591 INFO:teuthology.orchestra.run.smithi040.stderr:32800000 of 50000000 tuples (65%) done (elapsed 89.08 s, remaining 46.71 s) 2023-07-14T11:13:54.759 INFO:teuthology.orchestra.run.smithi040.stderr:32900000 of 50000000 tuples (65%) done (elapsed 89.25 s, remaining 46.39 s) 2023-07-14T11:13:54.894 INFO:teuthology.orchestra.run.smithi040.stderr:33000000 of 50000000 tuples (66%) done (elapsed 89.39 s, remaining 46.05 s) 2023-07-14T11:13:55.097 INFO:teuthology.orchestra.run.smithi040.stderr:33100000 of 50000000 tuples (66%) done (elapsed 89.59 s, remaining 45.74 s) 2023-07-14T11:13:55.300 INFO:teuthology.orchestra.run.smithi040.stderr:33200000 of 50000000 tuples (66%) done (elapsed 89.79 s, remaining 45.44 s) 2023-07-14T11:13:55.739 INFO:teuthology.orchestra.run.smithi040.stderr:33300000 of 50000000 tuples (66%) done (elapsed 90.23 s, remaining 45.25 s) 2023-07-14T11:13:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:55 smithi130 ceph-mon[129405]: pgmap v56: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 102 MiB/s wr, 152 op/s 2023-07-14T11:13:55.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:55 smithi040 ceph-mon[97954]: pgmap v56: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 102 MiB/s wr, 152 op/s 2023-07-14T11:13:55.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:55 smithi045 ceph-mon[128791]: pgmap v56: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 109 KiB/s rd, 102 MiB/s wr, 152 op/s 2023-07-14T11:13:56.027 INFO:teuthology.orchestra.run.smithi040.stderr:33400000 of 50000000 tuples (66%) done (elapsed 90.52 s, remaining 44.99 s) 2023-07-14T11:13:56.960 INFO:teuthology.orchestra.run.smithi040.stderr:33500000 of 50000000 tuples (67%) done (elapsed 91.45 s, remaining 45.04 s) 2023-07-14T11:13:57.125 INFO:teuthology.orchestra.run.smithi040.stderr:33600000 of 50000000 tuples (67%) done (elapsed 91.62 s, remaining 44.72 s) 2023-07-14T11:13:57.332 INFO:teuthology.orchestra.run.smithi040.stderr:33700000 of 50000000 tuples (67%) done (elapsed 91.83 s, remaining 44.41 s) 2023-07-14T11:13:57.453 INFO:teuthology.orchestra.run.smithi040.stderr:33800000 of 50000000 tuples (67%) done (elapsed 91.95 s, remaining 44.07 s) 2023-07-14T11:13:57.622 INFO:teuthology.orchestra.run.smithi040.stderr:33900000 of 50000000 tuples (67%) done (elapsed 92.11 s, remaining 43.75 s) 2023-07-14T11:13:57.806 INFO:teuthology.orchestra.run.smithi040.stderr:34000000 of 50000000 tuples (68%) done (elapsed 92.30 s, remaining 43.43 s) 2023-07-14T11:13:57.934 INFO:teuthology.orchestra.run.smithi040.stderr:34100000 of 50000000 tuples (68%) done (elapsed 92.43 s, remaining 43.10 s) 2023-07-14T11:13:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:57 smithi130 ceph-mon[129405]: pgmap v57: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:57 smithi040 ceph-mon[97954]: pgmap v57: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:57 smithi045 ceph-mon[128791]: pgmap v57: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:13:58.168 INFO:teuthology.orchestra.run.smithi040.stderr:34200000 of 50000000 tuples (68%) done (elapsed 92.66 s, remaining 42.81 s) 2023-07-14T11:13:58.353 INFO:teuthology.orchestra.run.smithi040.stderr:34300000 of 50000000 tuples (68%) done (elapsed 92.85 s, remaining 42.50 s) 2023-07-14T11:13:58.571 INFO:teuthology.orchestra.run.smithi040.stderr:34400000 of 50000000 tuples (68%) done (elapsed 93.06 s, remaining 42.20 s) 2023-07-14T11:13:58.775 INFO:teuthology.orchestra.run.smithi040.stderr:34500000 of 50000000 tuples (69%) done (elapsed 93.27 s, remaining 41.90 s) 2023-07-14T11:13:58.899 INFO:teuthology.orchestra.run.smithi040.stderr:34600000 of 50000000 tuples (69%) done (elapsed 93.39 s, remaining 41.57 s) 2023-07-14T11:13:59.124 INFO:teuthology.orchestra.run.smithi040.stderr:34700000 of 50000000 tuples (69%) done (elapsed 93.62 s, remaining 41.28 s) 2023-07-14T11:13:59.302 INFO:teuthology.orchestra.run.smithi040.stderr:34800000 of 50000000 tuples (69%) done (elapsed 93.79 s, remaining 40.97 s) 2023-07-14T11:14:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:13:59 smithi130 ceph-mon[129405]: pgmap v58: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:14:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:13:59 smithi040 ceph-mon[97954]: pgmap v58: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:14:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:13:59 smithi045 ceph-mon[128791]: pgmap v58: 129 pgs: 129 active+clean; 5.7 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 83 MiB/s wr, 74 op/s 2023-07-14T11:14:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: pgmap v59: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 146 MiB/s wr, 106 op/s 2023-07-14T11:14:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:01 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: pgmap v59: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 146 MiB/s wr, 106 op/s 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:01 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: pgmap v59: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 64 KiB/s rd, 146 MiB/s wr, 106 op/s 2023-07-14T11:14:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:01 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:03 smithi130 ceph-mon[129405]: pgmap v60: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 86 MiB/s wr, 70 op/s 2023-07-14T11:14:04.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:03 smithi040 ceph-mon[97954]: pgmap v60: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 86 MiB/s wr, 70 op/s 2023-07-14T11:14:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:03 smithi045 ceph-mon[128791]: pgmap v60: 129 pgs: 129 active+clean; 6.2 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 86 MiB/s wr, 70 op/s 2023-07-14T11:14:05.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:05 smithi130 ceph-mon[129405]: pgmap v61: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 87 MiB/s wr, 175 op/s 2023-07-14T11:14:05.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:05 smithi040 ceph-mon[97954]: pgmap v61: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 87 MiB/s wr, 175 op/s 2023-07-14T11:14:05.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:05 smithi045 ceph-mon[128791]: pgmap v61: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 78 KiB/s rd, 87 MiB/s wr, 175 op/s 2023-07-14T11:14:06.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:06 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:06.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:06 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:06.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:06 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:07.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:07 smithi130 ceph-mon[129405]: pgmap v62: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:07.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:07 smithi040 ceph-mon[97954]: pgmap v62: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:07.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:07 smithi045 ceph-mon[128791]: pgmap v62: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:08.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:08 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:14:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:08 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:14:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:08 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:14:08.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:08 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:14:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:08 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:14:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:08 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:14:08.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:08 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:14:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:08 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:14:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:08 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:14:09.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:09 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:14:09.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:09 smithi130 ceph-mon[129405]: scrub summary: idle 2023-07-14T11:14:09.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:09 smithi130 ceph-mon[129405]: pgmap v63: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:09.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:09 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:14:09.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:09 smithi040 ceph-mon[97954]: scrub summary: idle 2023-07-14T11:14:09.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:09 smithi040 ceph-mon[97954]: pgmap v63: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:09.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:09 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:14:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:09 smithi045 ceph-mon[128791]: scrub summary: idle 2023-07-14T11:14:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:09 smithi045 ceph-mon[128791]: pgmap v63: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 69 KiB/s rd, 64 MiB/s wr, 136 op/s 2023-07-14T11:14:10.406 INFO:teuthology.orchestra.run.smithi040.stderr:34900000 of 50000000 tuples (69%) done (elapsed 104.90 s, remaining 45.39 s) 2023-07-14T11:14:11.674 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:14:12.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 -- 172.21.15.40:0/2130360772 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 msgr2=0x7f8d940febf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 --2- 172.21.15.40:0/2130360772 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d940febf0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f8d84009990 tx=0x7f8d84009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 -- 172.21.15.40:0/2130360772 shutdown_connections 2023-07-14T11:14:12.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 --2- 172.21.15.40:0/2130360772 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d9410c3d0 0x7f8d9410e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 --2- 172.21.15.40:0/2130360772 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d940febf0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 --2- 172.21.15.40:0/2130360772 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d940ff1c0 0x7f8d9410be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 -- 172.21.15.40:0/2130360772 >> 172.21.15.40:0/2130360772 conn(0x7f8d940fa4b0 msgr2=0x7f8d940fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 -- 172.21.15.40:0/2130360772 shutdown_connections 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.002+0000 7f8d98bff700 1 -- 172.21.15.40:0/2130360772 wait complete. 2023-07-14T11:14:12.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.004+0000 7f8d98bff700 1 Processor -- start 2023-07-14T11:14:12.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.004+0000 7f8d98bff700 1 -- start start 2023-07-14T11:14:12.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d940ff1c0 0x7f8d941982e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d9410c3d0 0x7f8d9419c740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d9419cd80 con 0x7f8d9410c3d0 2023-07-14T11:14:12.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d9419cef0 con 0x7f8d940ff1c0 2023-07-14T11:14:12.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.005+0000 7f8d98bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8d9419d0b0 con 0x7f8d940fe820 2023-07-14T11:14:12.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59354/0 (socket says 172.21.15.40:59354) 2023-07-14T11:14:12.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 -- 172.21.15.40:0/16125631 learned_addr learned my addr 172.21.15.40:0/16125631 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d940ff1c0 0x7f8d941982e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d9410c3d0 0x7f8d9419c740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 -- 172.21.15.40:0/16125631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d940ff1c0 msgr2=0x7f8d941982e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d940ff1c0 0x7f8d941982e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 -- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d9410c3d0 msgr2=0x7f8d9419c740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d9410c3d0 0x7f8d9419c740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.006+0000 7f8d9259c700 1 -- 172.21.15.40:0/16125631 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8d840095f0 con 0x7f8d940fe820 2023-07-14T11:14:12.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d9259c700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f8d840095c0 tx=0x7f8d8400f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d837fe700 1 -- 172.21.15.40:0/16125631 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8401d070 con 0x7f8d940fe820 2023-07-14T11:14:12.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8d9419d330 con 0x7f8d940fe820 2023-07-14T11:14:12.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d837fe700 1 -- 172.21.15.40:0/16125631 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8d84005030 con 0x7f8d940fe820 2023-07-14T11:14:12.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d837fe700 1 -- 172.21.15.40:0/16125631 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d84004a20 con 0x7f8d940fe820 2023-07-14T11:14:12.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.007+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d941a4bd0 con 0x7f8d940fe820 2023-07-14T11:14:12.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.011+0000 7f8d837fe700 1 -- 172.21.15.40:0/16125631 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f8d84005410 con 0x7f8d940fe820 2023-07-14T11:14:12.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.013+0000 7f8dac484b80 1 Processor -- start 2023-07-14T11:14:12.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.013+0000 7f8dac484b80 1 -- start start 2023-07-14T11:14:12.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fc76a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe5ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3ecdcb0 con 0x5635a3fc72d0 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3f20a40 con 0x5635a3fe3ae0 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8dac484b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3f19ed0 con 0x5635a3fc7be0 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59364/0 (socket says 172.21.15.40:59364) 2023-07-14T11:14:12.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.014+0000 7f8d9259c700 1 -- 172.21.15.40:0/314629393 learned_addr learned my addr 172.21.15.40:0/314629393 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fc76a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe5ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 -- 172.21.15.40:0/314629393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 msgr2=0x5635a3fe35a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 -- 172.21.15.40:0/314629393 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 msgr2=0x5635a3fe5ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe5ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.015+0000 7f8d92d9d700 1 -- 172.21.15.40:0/314629393 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5635a3d47de0 con 0x5635a3fc72d0 2023-07-14T11:14:12.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.016+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fc76a0 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f8d94069c40 tx=0x7f8d8c009ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=28d5904e16944dbb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d837fe700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 0x7f8d7c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d9259c700 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:12.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d80ff9700 1 -- 172.21.15.40:0/314629393 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8c019070 con 0x5635a3fc72d0 2023-07-14T11:14:12.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d837fe700 1 -- 172.21.15.40:0/16125631 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8d840cde60 con 0x7f8d940fe820 2023-07-14T11:14:12.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d80ff9700 1 -- 172.21.15.40:0/314629393 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8d8c00f040 con 0x5635a3fc72d0 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d80ff9700 1 -- 172.21.15.40:0/314629393 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8c007870 con 0x5635a3fc72d0 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.017+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 0x7f8d7c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 -- 172.21.15.40:0/314629393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 msgr2=0x5635a3fc76a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fc76a0 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 crypto rx=0x7f8d94069c40 tx=0x7f8d8c009ba0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 -- 172.21.15.40:0/314629393 shutdown_connections 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe5ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fc76a0 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 --2- 172.21.15.40:0/314629393 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc7be0 0x5635a3fe35a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 -- 172.21.15.40:0/314629393 >> 172.21.15.40:0/314629393 conn(0x5635a3fbdda0 msgr2=0x5635a3fc36a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8dac484b80 1 -- 172.21.15.40:0/314629393 shutdown_connections 2023-07-14T11:14:12.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.018+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 0x7f8d7c0a3770 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f8d8c018000 tx=0x7f8d8c007c80 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.019+0000 7f8dac484b80 1 -- 172.21.15.40:0/314629393 wait complete. 2023-07-14T11:14:12.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.019+0000 7f8dac484b80 1 -- start start 2023-07-14T11:14:12.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.019+0000 7f8dac484b80 1 Processor -- start 2023-07-14T11:14:12.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fe0890 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 0x5635a3fe0dd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe1310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3eb9ac0 con 0x5635a3fc72d0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3eb97c0 con 0x5635a3fc7be0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3d9d920 con 0x5635a3fe3ae0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.020+0000 7f8dac484b80 10 client.0 _ll_register_callbacks cb 0x5635a3eb7760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.021+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fe0890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.021+0000 7f8d91d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 0x5635a3fe0dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.021+0000 7f8d91d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 0x5635a3fe0dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36526/0 (socket says 172.21.15.40:36526) 2023-07-14T11:14:12.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.021+0000 7f8d91d9b700 1 -- 172.21.15.40:0/100767547 learned_addr learned my addr 172.21.15.40:0/100767547 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 -- 172.21.15.40:0/100767547 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 msgr2=0x5635a3fe1310 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe1310 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 -- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 msgr2=0x5635a3fe0dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 0x5635a3fe0dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 -- 172.21.15.40:0/100767547 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d941a4bd0 con 0x5635a3fc72d0 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d9259c700 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fe0890 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f8d8409a990 tx=0x7f8d840cb2a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e285e213fd675046 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.022+0000 7f8d79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:14:12.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8409b0f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8d8409ac90 con 0x5635a3fc72d0 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5635a3e734f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8d8409a5f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8dac484b80 10 client.16086 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x7f8d8c003ae0 con 0x5635a3fc72d0 2023-07-14T11:14:12.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.023+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5635a3e734f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f8d840a2bc0 con 0x5635a3fc72d0 2023-07-14T11:14:12.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 10 client.16086 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:12.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 10 client.16086 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:12.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 20 client.16086 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:12.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3ffb7d0 0x5635a3ffdbe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 4 client.16086 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:12.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.024+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5635a3f12eb0 con 0x5635a3ffb7d0 2023-07-14T11:14:12.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.026+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3ffb7d0 0x5635a3ffdbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.026+0000 7f8d79ffb700 10 client.16086 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:12.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.026+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3ffb7d0 0x5635a3ffdbe0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6858c5181685f437 server_cookie=53f282f65d1f5b76 in_seq=0 out_seq=0 2023-07-14T11:14:12.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.027+0000 7f8d79ffb700 1 -- 172.21.15.40:0/100767547 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5635a3f12eb0 con 0x5635a3ffb7d0 2023-07-14T11:14:12.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.027+0000 7f8d79ffb700 10 client.16086 handle_command_reply: tid=0 2023-07-14T11:14:12.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.027+0000 7f8dac484b80 1 client.16086 shutdown 2023-07-14T11:14:12.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 msgr2=0x5635a3fe0890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fe0890 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 crypto rx=0x7f8d8409a990 tx=0x7f8d840cb2a0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 shutdown_connections 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3ffb7d0 0x5635a3ffdbe0 crc :-1 s=READY pgs=73 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fe1310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8d79ffb700 0 client.16086 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc72d0 0x5635a3fe0890 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 --2- 172.21.15.40:0/100767547 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc7be0 0x5635a3fe0dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 >> 172.21.15.40:0/100767547 conn(0x5635a3fbdda0 msgr2=0x5635a3fc3230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 shutdown_connections 2023-07-14T11:14:12.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 1 -- 172.21.15.40:0/100767547 wait complete. 2023-07-14T11:14:12.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.028+0000 7f8dac484b80 20 client.16086 trim_cache size 0 max 16384 2023-07-14T11:14:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:11 smithi130 ceph-mon[129405]: pgmap v64: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 65 MiB/s wr, 265 op/s 2023-07-14T11:14:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:11 smithi040 ceph-mon[97954]: pgmap v64: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 65 MiB/s wr, 265 op/s 2023-07-14T11:14:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:11 smithi045 ceph-mon[128791]: pgmap v64: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 80 KiB/s rd, 65 MiB/s wr, 265 op/s 2023-07-14T11:14:12.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.065+0000 7f8dac484b80 1 Processor -- start 2023-07-14T11:14:12.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 -- start start 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 0x5635a3fe30a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff2cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fddb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3e734f0 con 0x5635a3fc7be0 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3fe38a0 con 0x5635a3fc72d0 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.066+0000 7f8dac484b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3fe1860 con 0x5635a3fe3ae0 2023-07-14T11:14:12.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d92d9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fddb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 0x5635a3fe30a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d92d9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fddb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59382/0 (socket says 172.21.15.40:59382) 2023-07-14T11:14:12.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d92d9d700 1 -- 172.21.15.40:0/4235170043 learned_addr learned my addr 172.21.15.40:0/4235170043 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff2cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 -- 172.21.15.40:0/4235170043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 msgr2=0x5635a3fddb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fddb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 -- 172.21.15.40:0/4235170043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 msgr2=0x5635a3ff2cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff2cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 -- 172.21.15.40:0/4235170043 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5635a3fe0400 con 0x5635a3fc72d0 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.067+0000 7f8d9259c700 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 0x5635a3fe30a0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f8d840a2b20 tx=0x7f8d840ce130 comp rx=0 tx=0).ready entity=mon.2 client_cookie=354c5d657bdf6ed server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.068+0000 7f8d817fa700 1 -- 172.21.15.40:0/4235170043 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8409a5f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.068+0000 7f8d817fa700 1 -- 172.21.15.40:0/4235170043 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8d8409e060 con 0x5635a3fc72d0 2023-07-14T11:14:12.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.068+0000 7f8d817fa700 1 -- 172.21.15.40:0/4235170043 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d840a3050 con 0x5635a3fc72d0 2023-07-14T11:14:12.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 -- 172.21.15.40:0/4235170043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 msgr2=0x5635a3fe30a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 0x5635a3fe30a0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f8d840a2b20 tx=0x7f8d840ce130 comp rx=0 tx=0).stop 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 -- 172.21.15.40:0/4235170043 shutdown_connections 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fe3ae0 0x5635a3fddb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fc72d0 0x5635a3fe30a0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 --2- 172.21.15.40:0/4235170043 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff2cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.069+0000 7f8dac484b80 1 -- 172.21.15.40:0/4235170043 >> 172.21.15.40:0/4235170043 conn(0x5635a3fbdda0 msgr2=0x5635a3ff7fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.070+0000 7f8dac484b80 1 -- 172.21.15.40:0/4235170043 shutdown_connections 2023-07-14T11:14:12.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.070+0000 7f8dac484b80 1 -- 172.21.15.40:0/4235170043 wait complete. 2023-07-14T11:14:12.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.070+0000 7f8dac484b80 1 -- start start 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.071+0000 7f8dac484b80 1 Processor -- start 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff1120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3ff16e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3ff1df0 con 0x5635a3fc7be0 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3ff1f60 con 0x5635a3fe3ae0 2023-07-14T11:14:12.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5635a3fe6400 con 0x5635a3fc72d0 2023-07-14T11:14:12.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.072+0000 7f8dac484b80 10 client.0 _ll_register_callbacks cb 0x7f8d840d1600 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:12.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59388/0 (socket says 172.21.15.40:59388) 2023-07-14T11:14:12.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 -- 172.21.15.40:0/2336129862 learned_addr learned my addr 172.21.15.40:0/2336129862 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d92d9d700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3ff16e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff1120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 -- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 msgr2=0x5635a3ff16e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.077 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3ff16e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 -- 172.21.15.40:0/2336129862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 msgr2=0x5635a3ff1120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff1120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d9259c700 1 -- 172.21.15.40:0/2336129862 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8d8c003ae0 con 0x5635a3fc72d0 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.073+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff1120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.074+0000 7f8d9259c700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8d8409ac20 tx=0x7f8d840a2df0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a474f180e1654686 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.074+0000 7f8d79ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.074+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8d8409b0f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.078 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.074+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f8d840a3050 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.074+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8d8c00f040 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8d8409a5f0 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 10 client.26074 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x5635a3d30c60 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8d8c00f040 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f8d84004ca0 con 0x5635a3fc72d0 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 10 client.26074 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:12.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 10 client.26074 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:12.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 20 client.26074 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:12.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3fe77d0 0x5635a3fe9be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 4 client.26074 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:14:12.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.075+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5635a3ed5050 con 0x5635a3fe77d0 2023-07-14T11:14:12.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.076+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3fe77d0 0x5635a3fe9be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.076+0000 7f8d91d9b700 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3fe77d0 0x5635a3fe9be0 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f8aa8f4da8a627c1 server_cookie=27a1d2f1c3b93a42 in_seq=0 out_seq=0 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.077+0000 7f8d79ffb700 10 client.26074 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.077+0000 7f8d79ffb700 1 -- 172.21.15.40:0/2336129862 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+63 (crc 0 0 0) 0x5635a3ed5050 con 0x5635a3fe77d0 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.077+0000 7f8d79ffb700 10 client.26074 handle_command_reply: tid=0 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.077+0000 7f8dac484b80 1 client.26074 shutdown 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 msgr2=0x5635a3ff2df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f8d8409ac20 tx=0x7f8d840a2df0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 shutdown_connections 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5635a3fe77d0 0x5635a3fe9be0 crc :-1 s=READY pgs=74 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5635a3fe3ae0 0x5635a3ff16e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5635a3fc72d0 0x5635a3ff2df0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 --2- 172.21.15.40:0/2336129862 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5635a3fc7be0 0x5635a3ff1120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 >> 172.21.15.40:0/2336129862 conn(0x5635a3fbdda0 msgr2=0x5635a3fe07e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8d79ffb700 0 client.26074 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:12.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 shutdown_connections 2023-07-14T11:14:12.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.078+0000 7f8dac484b80 1 -- 172.21.15.40:0/2336129862 wait complete. 2023-07-14T11:14:12.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8dac484b80 20 client.26074 trim_cache size 0 max 16384 2023-07-14T11:14:12.084 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:14:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "no active scrubs running", 2023-07-14T11:14:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": {} 2023-07-14T11:14:12.085 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:14:12.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 msgr2=0x7f8d7c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 0x7f8d7c0a3770 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f8d8c018000 tx=0x7f8d8c007c80 comp rx=0 tx=0).stop 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 msgr2=0x7f8d94197da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f8d840095c0 tx=0x7f8d8400f690 comp rx=0 tx=0).stop 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 shutdown_connections 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8d7c0a12c0 0x7f8d7c0a3770 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8d9410c3d0 0x7f8d9419c740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8d940fe820 0x7f8d94197da0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 --2- 172.21.15.40:0/16125631 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8d940ff1c0 0x7f8d941982e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.079+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 >> 172.21.15.40:0/16125631 conn(0x7f8d940fa4b0 msgr2=0x7f8d9410ce20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.080+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 shutdown_connections 2023-07-14T11:14:12.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.080+0000 7f8d98bff700 1 -- 172.21.15.40:0/16125631 wait complete. 2023-07-14T11:14:12.112 INFO:tasks.cephfs.filesystem:all active scrubs completed 2023-07-14T11:14:12.112 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:14:12.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 -- 172.21.15.40:0/444429354 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc878110ae0 msgr2=0x7fc878112ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 --2- 172.21.15.40:0/444429354 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc878110ae0 0x7fc878112ec0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7fc87000b0a0 tx=0x7fc87000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 -- 172.21.15.40:0/444429354 shutdown_connections 2023-07-14T11:14:12.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 --2- 172.21.15.40:0/444429354 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc878110ae0 0x7fc878112ec0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 --2- 172.21.15.40:0/444429354 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780728b0 0x7fc878072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 --2- 172.21.15.40:0/444429354 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc8780731c0 0x7fc8781105a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.411+0000 7fc87cfff700 1 -- 172.21.15.40:0/444429354 >> 172.21.15.40:0/444429354 conn(0x7fc8780fb590 msgr2=0x7fc8780fd9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.412+0000 7fc87cfff700 1 -- 172.21.15.40:0/444429354 shutdown_connections 2023-07-14T11:14:12.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.412+0000 7fc87cfff700 1 -- 172.21.15.40:0/444429354 wait complete. 2023-07-14T11:14:12.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.413+0000 7fc87cfff700 1 Processor -- start 2023-07-14T11:14:12.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.413+0000 7fc87cfff700 1 -- start start 2023-07-14T11:14:12.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8780728b0 0x7fc87819c180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 0x7fc8781a0b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8781a1160 con 0x7fc8780731c0 2023-07-14T11:14:12.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8781a12d0 con 0x7fc878110ae0 2023-07-14T11:14:12.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc87cfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc8781a1490 con 0x7fc8780728b0 2023-07-14T11:14:12.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc875d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc875d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56670/0 (socket says 172.21.15.40:56670) 2023-07-14T11:14:12.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc875d9b700 1 -- 172.21.15.40:0/3484695434 learned_addr learned my addr 172.21.15.40:0/3484695434 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:12.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.414+0000 7fc876d9d700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 0x7fc8781a0b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 -- 172.21.15.40:0/3484695434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8780728b0 msgr2=0x7fc87819c180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8780728b0 0x7fc87819c180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 -- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 msgr2=0x7fc87819c6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 -- 172.21.15.40:0/3484695434 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc870009d00 con 0x7fc878110ae0 2023-07-14T11:14:12.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc875d9b700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc876d9d700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 0x7fc8781a0b20 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fc870009fd0 tx=0x7fc870009000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc870021070 con 0x7fc878110ae0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc8700075c0 con 0x7fc878110ae0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.415+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc8781a1710 con 0x7fc878110ae0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.416+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc87000e040 con 0x7fc878110ae0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.416+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc8781a8fb0 con 0x7fc878110ae0 2023-07-14T11:14:12.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.417+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc878102fc0 con 0x7fc878110ae0 2023-07-14T11:14:12.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.420+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fc870007730 con 0x7fc878110ae0 2023-07-14T11:14:12.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.422+0000 7fc86f7fe700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 0x7fc8640a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:12.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.422+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fc8700cddb0 con 0x7fc878110ae0 2023-07-14T11:14:12.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.423+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc870096ee0 con 0x7fc878110ae0 2023-07-14T11:14:12.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.423+0000 7fc87659c700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 0x7fc8640a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:12.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.424+0000 7fc87659c700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 0x7fc8640a3870 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc868005950 tx=0x7fc8680058e0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:12.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.863+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7fc878067300 con 0x7fc878110ae0 2023-07-14T11:14:12.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.864+0000 7fc86f7fe700 1 -- 172.21.15.40:0/3484695434 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 35 v35) v1 ==== 94+0+10907 (secure 0 0 0) 0x7fc87009a500 con 0x7fc878110ae0 2023-07-14T11:14:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:14:12.866 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":35,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":35,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:14:05.430135+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[0,1,3],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":11,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[0,2,4],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":3,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[1,2,3,4],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:14:12.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.868+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 msgr2=0x7fc8640a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.868+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 0x7fc8640a3870 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc868005950 tx=0x7fc8680058e0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.868+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 msgr2=0x7fc8781a0b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:12.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.868+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 0x7fc8781a0b20 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7fc870009fd0 tx=0x7fc870009000 comp rx=0 tx=0).stop 2023-07-14T11:14:12.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 shutdown_connections 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc8640a13c0 0x7fc8640a3870 secure :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fc868005950 tx=0x7fc8680058e0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc878110ae0 0x7fc8781a0b20 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc8780728b0 0x7fc87819c180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 --2- 172.21.15.40:0/3484695434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8780731c0 0x7fc87819c6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 >> 172.21.15.40:0/3484695434 conn(0x7fc8780fb590 msgr2=0x7fc8780fc1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:12.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 shutdown_connections 2023-07-14T11:14:12.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:12.869+0000 7fc87cfff700 1 -- 172.21.15.40:0/3484695434 wait complete. 2023-07-14T11:14:12.873 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 35 2023-07-14T11:14:12.898 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:4 damage ls 2023-07-14T11:14:13.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 -- 172.21.15.40:0/292574144 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 msgr2=0x7f7a9810c4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/292574144 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a9810c4b0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f7a9000b0a0 tx=0x7f7a9000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 -- 172.21.15.40:0/292574144 shutdown_connections 2023-07-14T11:14:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/292574144 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a9810c4b0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/292574144 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a98100ca0 0x7f7a98101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/292574144 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a981015b0 0x7f7a98109aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.227+0000 7f7a9fdff700 1 -- 172.21.15.40:0/292574144 >> 172.21.15.40:0/292574144 conn(0x7f7a980fa4b0 msgr2=0x7f7a980fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.228+0000 7f7a9fdff700 1 -- 172.21.15.40:0/292574144 shutdown_connections 2023-07-14T11:14:13.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.228+0000 7f7a9fdff700 1 -- 172.21.15.40:0/292574144 wait complete. 2023-07-14T11:14:13.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.230+0000 7f7a9fdff700 1 Processor -- start 2023-07-14T11:14:13.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.230+0000 7f7a9fdff700 1 -- start start 2023-07-14T11:14:13.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a98100ca0 0x7f7a98101b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a98102590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a981a85b0 con 0x7f7a98100ca0 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a981a8720 con 0x7f7a9810a070 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9fdff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7a981a88e0 con 0x7f7a981015b0 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9d39a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9d39a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59438/0 (socket says 172.21.15.40:59438) 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9d39a700 1 -- 172.21.15.40:0/3933858063 learned_addr learned my addr 172.21.15.40:0/3933858063 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a98100ca0 0x7f7a98101b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.231+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a98102590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 -- 172.21.15.40:0/3933858063 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 msgr2=0x7f7a98102590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a98102590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 -- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a98100ca0 msgr2=0x7f7a98101b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a98100ca0 0x7f7a98101b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 -- 172.21.15.40:0/3933858063 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7a90009d00 con 0x7f7a981015b0 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a98102590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f7a9400d980 tx=0x7f7a9400dc90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a8effd700 1 -- 172.21.15.40:0/3933858063 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a94004220 con 0x7f7a981015b0 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a8effd700 1 -- 172.21.15.40:0/3933858063 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a94004cf0 con 0x7f7a981015b0 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7a981a8bc0 con 0x7f7a981015b0 2023-07-14T11:14:13.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.232+0000 7f7a8effd700 1 -- 172.21.15.40:0/3933858063 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a9400f690 con 0x7f7a981015b0 2023-07-14T11:14:13.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.233+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a981a9120 con 0x7f7a981015b0 2023-07-14T11:14:13.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.238+0000 7f7a8effd700 1 -- 172.21.15.40:0/3933858063 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f7a94004750 con 0x7f7a981015b0 2023-07-14T11:14:13.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.240+0000 7f7ab369ab80 1 Processor -- start 2023-07-14T11:14:13.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.240+0000 7f7ab369ab80 1 -- start start 2023-07-14T11:14:13.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.240+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.240+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42efa590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42efaad0 0x55ec42efceb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42df5cb0 con 0x55ec42eef2d0 2023-07-14T11:14:13.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42e48a40 con 0x55ec42efaad0 2023-07-14T11:14:13.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42e41ed0 con 0x55ec42eefbe0 2023-07-14T11:14:13.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7a9e39c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.241+0000 7f7a9e39c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56692/0 (socket says 172.21.15.40:56692) 2023-07-14T11:14:13.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9e39c700 1 -- 172.21.15.40:0/1063932690 learned_addr learned my addr 172.21.15.40:0/1063932690 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42efa590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a8effd700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 0x7f7a840a3870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 -- 172.21.15.40:0/1063932690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42efaad0 msgr2=0x55ec42efceb0 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:14:13.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42efaad0 0x55ec42efceb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 -- 172.21.15.40:0/1063932690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 msgr2=0x55ec42eef6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9db9b700 1 -- 172.21.15.40:0/1063932690 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ec42c6fde0 con 0x55ec42eefbe0 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.242+0000 7f7a8effd700 1 -- 172.21.15.40:0/3933858063 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7a940ccd20 con 0x7f7a981015b0 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.243+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42efa590 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800ac60 tx=0x7f7a8800af70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8ca530532f7ccccd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.246 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.243+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 0x7f7a840a3870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.243+0000 7f7a83fff700 1 -- 172.21.15.40:0/1063932690 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a8800fd80 con 0x55ec42eefbe0 2023-07-14T11:14:13.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.243+0000 7f7a83fff700 1 -- 172.21.15.40:0/1063932690 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a880044f0 con 0x55ec42eefbe0 2023-07-14T11:14:13.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.243+0000 7f7a83fff700 1 -- 172.21.15.40:0/1063932690 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a88005280 con 0x55ec42eefbe0 2023-07-14T11:14:13.247 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 -- 172.21.15.40:0/1063932690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 msgr2=0x55ec42efa590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 0x7f7a840a3870 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f7a9000b0a0 tx=0x7f7a900099e0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42efa590 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800ac60 tx=0x7f7a8800af70 comp rx=0 tx=0).stop 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 -- 172.21.15.40:0/1063932690 shutdown_connections 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42efaad0 0x55ec42efceb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42eef6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.248 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/1063932690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42efa590 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 -- 172.21.15.40:0/1063932690 >> 172.21.15.40:0/1063932690 conn(0x55ec42ee5da0 msgr2=0x55ec42eeb6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 -- 172.21.15.40:0/1063932690 shutdown_connections 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.244+0000 7f7ab369ab80 1 -- 172.21.15.40:0/1063932690 wait complete. 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.245+0000 7f7ab369ab80 1 -- start start 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.245+0000 7f7ab369ab80 1 Processor -- start 2023-07-14T11:14:13.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42ef7830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42efaad0 0x55ec42ef7f20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42de1ac0 con 0x55ec42efaad0 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42de17c0 con 0x55ec42eef2d0 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42cc5920 con 0x55ec42eefbe0 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7ab369ab80 10 client.0 _ll_register_callbacks cb 0x55ec42ddf760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:13.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7a9db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7a9db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36606/0 (socket says 172.21.15.40:36606) 2023-07-14T11:14:13.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.247+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2621713559 learned_addr learned my addr 172.21.15.40:0/2621713559 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 msgr2=0x55ec42ef7830 unknown :-1 s=STATE_CONNECTING l=0).mark_down 2023-07-14T11:14:13.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42ef7830 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2621713559 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42efaad0 msgr2=0x55ec42ef7f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42efaad0 0x55ec42ef7f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2621713559 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a9810d170 con 0x55ec42eef2d0 2023-07-14T11:14:13.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800a600 tx=0x7f7a8800fed0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6742750cdb1d70a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a817fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:13.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a880053f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7a88005550 con 0x55ec42eef2d0 2023-07-14T11:14:13.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.248+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ec42d9b4f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.249+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7a88005710 con 0x55ec42eef2d0 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.250+0000 7f7ab369ab80 10 client.26114 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.250+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x7f7a8800a890 con 0x55ec42eef2d0 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.250+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ec42d9b4f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.250+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f7a88010530 con 0x55ec42eef2d0 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 10 client.26114 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:13.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 10 client.26114 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:14:13.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 20 client.26114 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:13.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f0a530 0x55ec42f0c940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 4 client.26114 mds_command: new command op to 24511 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:13.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ec42e3aeb0 con 0x55ec42f0a530 2023-07-14T11:14:13.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.251+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f0a530 0x55ec42f0c940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.253+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f0a530 0x55ec42f0c940 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=f65b742702e3571c server_cookie=bbb89207b6171c0f in_seq=0 out_seq=0 2023-07-14T11:14:13.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.253+0000 7f7a817fa700 10 client.26114 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:14:13.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7a817fa700 1 -- 172.21.15.40:0/2621713559 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55ec42e3aeb0 con 0x55ec42f0a530 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7a817fa700 10 client.26114 handle_command_reply: tid=0 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7ab369ab80 1 client.26114 shutdown 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 msgr2=0x55ec42efa540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800a600 tx=0x7f7a8800fed0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 shutdown_connections 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.254+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f0a530 0x55ec42f0c940 crc :-1 s=READY pgs=41 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42efaad0 0x55ec42ef7f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa540 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2621713559 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42eefbe0 0x55ec42ef7830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 >> 172.21.15.40:0/2621713559 conn(0x55ec42ee5da0 msgr2=0x55ec42eeb290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7a817fa700 0 client.26114 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 shutdown_connections 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2621713559 wait complete. 2023-07-14T11:14:13.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.255+0000 7f7ab369ab80 20 client.26114 trim_cache size 0 max 16384 2023-07-14T11:14:13.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.277+0000 7f7ab369ab80 1 Processor -- start 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.277+0000 7f7ab369ab80 1 -- start start 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eefbe0 0x55ec42ef2af0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42ef3030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42ef01c0 con 0x55ec42eefbe0 2023-07-14T11:14:13.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42ee5970 con 0x55ec42eef2d0 2023-07-14T11:14:13.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42ef0dd0 con 0x55ec42efaad0 2023-07-14T11:14:13.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7a9db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7a9db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36612/0 (socket says 172.21.15.40:36612) 2023-07-14T11:14:13.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2447323291 learned_addr learned my addr 172.21.15.40:0/2447323291 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eefbe0 0x55ec42ef2af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.278+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42ef3030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2447323291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 msgr2=0x55ec42ef3030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42ef3030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2447323291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eefbe0 msgr2=0x55ec42ef2af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eefbe0 0x55ec42ef2af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 -- 172.21.15.40:0/2447323291 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ec42c58c60 con 0x55ec42eef2d0 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.279+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f7a8803f040 tx=0x7f7a88005f60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bd8972559f3fde78 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.280+0000 7f7a8cbff700 1 -- 172.21.15.40:0/2447323291 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a88010e40 con 0x55ec42eef2d0 2023-07-14T11:14:13.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.280+0000 7f7a8cbff700 1 -- 172.21.15.40:0/2447323291 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a8800a380 con 0x55ec42eef2d0 2023-07-14T11:14:13.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.280+0000 7f7a8cbff700 1 -- 172.21.15.40:0/2447323291 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a88010630 con 0x55ec42eef2d0 2023-07-14T11:14:13.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.280+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2447323291 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 msgr2=0x55ec42efa0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.280+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f7a8803f040 tx=0x7f7a88005f60 comp rx=0 tx=0).stop 2023-07-14T11:14:13.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2447323291 shutdown_connections 2023-07-14T11:14:13.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42ef3030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eef2d0 0x55ec42efa0e0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/2447323291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eefbe0 0x55ec42ef2af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2447323291 >> 172.21.15.40:0/2447323291 conn(0x55ec42ee5da0 msgr2=0x55ec42f05d20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2447323291 shutdown_connections 2023-07-14T11:14:13.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.281+0000 7f7ab369ab80 1 -- 172.21.15.40:0/2447323291 wait complete. 2023-07-14T11:14:13.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 -- start start 2023-07-14T11:14:13.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 Processor -- start 2023-07-14T11:14:13.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42efefe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 0x55ec42eff520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42f5d3b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42effde0 con 0x55ec42eef2d0 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42f5d8f0 con 0x55ec42eefbe0 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.282+0000 7f7ab369ab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ec42f5da30 con 0x55ec42efaad0 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7ab369ab80 10 client.0 _ll_register_callbacks cb 0x7f7a88005eb0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7a9d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 0x55ec42eff520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7a9d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 0x55ec42eff520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36616/0 (socket says 172.21.15.40:36616) 2023-07-14T11:14:13.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7a9d39a700 1 -- 172.21.15.40:0/548490607 learned_addr learned my addr 172.21.15.40:0/548490607 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42f5d3b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.283+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42efefe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 -- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 msgr2=0x55ec42f5d3b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42f5d3b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 -- 172.21.15.40:0/548490607 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 msgr2=0x55ec42eff520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 0x55ec42eff520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 -- 172.21.15.40:0/548490607 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7a8800a890 con 0x55ec42eef2d0 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9e39c700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42f5d3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.284+0000 7f7a9db9b700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42efefe0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800fbc0 tx=0x7f7a8800fed0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ce5e31b4daad617e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.285+0000 7f7a817fa700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.285+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7a880051f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.285+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7a88010e40 con 0x55ec42eef2d0 2023-07-14T11:14:13.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.285+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7a880044f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.286+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7a88005350 con 0x55ec42eef2d0 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.286+0000 7f7ab369ab80 10 client.16110 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.286+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x55ec42c96250 con 0x55ec42eef2d0 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7a880044f0 con 0x55ec42eef2d0 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f7a8804fc70 con 0x55ec42eef2d0 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 10 client.16110 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:13.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 10 client.16110 resolve_mds: resolved 1:4 to role '1:4' aka daemon mds.f 2023-07-14T11:14:13.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 20 client.16110 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:13.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f5ebf0 0x55ec42f60fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 4 client.16110 mds_command: new command op to 24511 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:14:13.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 --> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55ec42dfd050 con 0x55ec42f5ebf0 2023-07-14T11:14:13.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.287+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f5ebf0 0x55ec42f60fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.289+0000 7f7a9d39a700 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f5ebf0 0x55ec42f60fd0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.4 client_cookie=e4da91c27fc8669b server_cookie=c6fb5774226a7ce in_seq=0 out_seq=0 2023-07-14T11:14:13.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.289+0000 7f7a817fa700 10 client.16110 ms_handle_connect on v2:172.21.15.130:6834/2228846875 2023-07-14T11:14:13.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.290+0000 7f7a817fa700 1 -- 172.21.15.40:0/548490607 <== mds.4 v2:172.21.15.130:6834/2228846875 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55ec42dfd050 con 0x55ec42f5ebf0 2023-07-14T11:14:13.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.290+0000 7f7a817fa700 10 client.16110 handle_command_reply: tid=0 2023-07-14T11:14:13.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.290+0000 7f7ab369ab80 1 client.16110 shutdown 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 msgr2=0x55ec42efefe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42efefe0 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 crypto rx=0x7f7a8800fbc0 tx=0x7f7a8800fed0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 shutdown_connections 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:6834/2228846875,v1:172.21.15.130:6835/2228846875] conn(0x55ec42f5ebf0 0x55ec42f60fd0 crc :-1 s=READY pgs=42 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ec42efaad0 0x55ec42f5d3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ec42eef2d0 0x55ec42efefe0 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 --2- 172.21.15.40:0/548490607 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ec42eefbe0 0x55ec42eff520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7a817fa700 0 client.16110 ms_handle_reset on v2:172.21.15.130:6834/2228846875 2023-07-14T11:14:13.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 >> 172.21.15.40:0/548490607 conn(0x55ec42ee5da0 msgr2=0x55ec42ef2af0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 shutdown_connections 2023-07-14T11:14:13.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.291+0000 7f7ab369ab80 1 -- 172.21.15.40:0/548490607 wait complete. 2023-07-14T11:14:13.296 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:14:13.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.292+0000 7f7ab369ab80 20 client.16110 trim_cache size 0 max 16384 2023-07-14T11:14:13.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.293+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 msgr2=0x7f7a840a3870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.293+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 0x7f7a840a3870 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f7a9000b0a0 tx=0x7f7a900099e0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.293+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 msgr2=0x7f7a98102050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.293+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f7a9400d980 tx=0x7f7a9400dc90 comp rx=0 tx=0).stop 2023-07-14T11:14:13.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 shutdown_connections 2023-07-14T11:14:13.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7a840a13c0 0x7f7a840a3870 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7a9810a070 0x7f7a98102590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7a98100ca0 0x7f7a98101b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 --2- 172.21.15.40:0/3933858063 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7a981015b0 0x7f7a98102050 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.294+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 >> 172.21.15.40:0/3933858063 conn(0x7f7a980fa4b0 msgr2=0x7f7a980fc890 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.295+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 shutdown_connections 2023-07-14T11:14:13.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.295+0000 7f7a9fdff700 1 -- 172.21.15.40:0/3933858063 wait complete. 2023-07-14T11:14:13.322 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:3 damage ls 2023-07-14T11:14:13.622 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:13 smithi040 ceph-mon[97954]: pgmap v65: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 1.8 MiB/s wr, 233 op/s 2023-07-14T11:14:13.622 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:13 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3484695434' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:14:13.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.653+0000 7f69e21ff700 1 -- 172.21.15.40:0/1790830100 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc100ca0 msgr2=0x7f69dc101070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.653+0000 7f69e21ff700 1 --2- 172.21.15.40:0/1790830100 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc100ca0 0x7f69dc101070 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7f69cc009990 tx=0x7f69cc009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 -- 172.21.15.40:0/1790830100 shutdown_connections 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 --2- 172.21.15.40:0/1790830100 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc10a070 0x7f69dc10c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 --2- 172.21.15.40:0/1790830100 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc100ca0 0x7f69dc101070 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 --2- 172.21.15.40:0/1790830100 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc109aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 -- 172.21.15.40:0/1790830100 >> 172.21.15.40:0/1790830100 conn(0x7f69dc0fa4b0 msgr2=0x7f69dc0fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 -- 172.21.15.40:0/1790830100 shutdown_connections 2023-07-14T11:14:13.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.654+0000 7f69e21ff700 1 -- 172.21.15.40:0/1790830100 wait complete. 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.655+0000 7f69e21ff700 1 Processor -- start 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.655+0000 7f69e21ff700 1 -- start start 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc100ca0 0x7f69dc197e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 0x7f69dc19c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69dc19ce00 con 0x7f69dc100ca0 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69dc19cf70 con 0x7f69dc10a070 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69e21ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69dc19d130 con 0x7f69dc1015b0 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69dbfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 0x7f69dc19c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69daffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60560/0 (socket says 172.21.15.40:60560) 2023-07-14T11:14:13.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.656+0000 7f69daffd700 1 -- 172.21.15.40:0/2948460134 learned_addr learned my addr 172.21.15.40:0/2948460134 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc100ca0 0x7f69dc197e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 -- 172.21.15.40:0/2948460134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 msgr2=0x7f69dc198360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 -- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc100ca0 msgr2=0x7f69dc197e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc100ca0 0x7f69dc197e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 -- 172.21.15.40:0/2948460134 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f69cc0095f0 con 0x7f69dc10a070 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69daffd700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69dbfff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 0x7f69dc19c7c0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f69d400cc50 tx=0x7f69d400cf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69d8ff9700 1 -- 172.21.15.40:0/2948460134 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69d400ab40 con 0x7f69dc10a070 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69dc19d410 con 0x7f69dc10a070 2023-07-14T11:14:13.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69d8ff9700 1 -- 172.21.15.40:0/2948460134 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69d400aca0 con 0x7f69dc10a070 2023-07-14T11:14:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69d8ff9700 1 -- 172.21.15.40:0/2948460134 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69d40115e0 con 0x7f69dc10a070 2023-07-14T11:14:13.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.657+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69dc1a4c50 con 0x7f69dc10a070 2023-07-14T11:14:13.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.662+0000 7f69d8ff9700 1 -- 172.21.15.40:0/2948460134 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f69d4012420 con 0x7f69dc10a070 2023-07-14T11:14:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.663+0000 7f69f5a75b80 1 Processor -- start 2023-07-14T11:14:13.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 -- start start 2023-07-14T11:14:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6db6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e8eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e5e1cb0 con 0x562f0e6e6ad0 2023-07-14T11:14:13.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e634a40 con 0x562f0e6db2d0 2023-07-14T11:14:13.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e62ded0 con 0x562f0e6dbbe0 2023-07-14T11:14:13.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69daffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6db6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60566/0 (socket says 172.21.15.40:60566) 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.664+0000 7f69db7fe700 1 -- 172.21.15.40:0/584798502 learned_addr learned my addr 172.21.15.40:0/584798502 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e8eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 -- 172.21.15.40:0/584798502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 msgr2=0x562f0e6e6590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 -- 172.21.15.40:0/584798502 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 msgr2=0x562f0e6e8eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e8eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 -- 172.21.15.40:0/584798502 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f0e45bde0 con 0x562f0e6db2d0 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69d8ff9700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 0x7f69c40a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69d8ff9700 1 -- 172.21.15.40:0/2948460134 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f69d40ccfe0 con 0x7f69dc10a070 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69daffd700 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6db6a0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f69d00096b0 tx=0x7f69d0009cf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=2b6cae032a7685b5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.665+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 0x7f69c40a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.666+0000 7f69c1ffb700 1 -- 172.21.15.40:0/584798502 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69d0004030 con 0x562f0e6db2d0 2023-07-14T11:14:13.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.666+0000 7f69c1ffb700 1 -- 172.21.15.40:0/584798502 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69d0017070 con 0x562f0e6db2d0 2023-07-14T11:14:13.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.666+0000 7f69c1ffb700 1 -- 172.21.15.40:0/584798502 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69d0005280 con 0x562f0e6db2d0 2023-07-14T11:14:13.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 0x7f69c40a37f0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f69cc006010 tx=0x7f69cc01b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 -- 172.21.15.40:0/584798502 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 msgr2=0x562f0e6db6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6db6a0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f69d00096b0 tx=0x7f69d0009cf0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 -- 172.21.15.40:0/584798502 shutdown_connections 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e8eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6db6a0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/584798502 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 -- 172.21.15.40:0/584798502 >> 172.21.15.40:0/584798502 conn(0x562f0e6d1da0 msgr2=0x562f0e6d76a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 -- 172.21.15.40:0/584798502 shutdown_connections 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.667+0000 7f69f5a75b80 1 -- 172.21.15.40:0/584798502 wait complete. 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.668+0000 7f69f5a75b80 1 -- start start 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.668+0000 7f69f5a75b80 1 Processor -- start 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6e64d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e36e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e5cdac0 con 0x562f0e6dbbe0 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e5cd7c0 con 0x562f0e6e6ad0 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e4b1920 con 0x562f0e6db2d0 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.669+0000 7f69f5a75b80 10 client.0 _ll_register_callbacks cb 0x562f0e5cb760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34658/0 (socket says 172.21.15.40:34658) 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 -- 172.21.15.40:0/4223734334 learned_addr learned my addr 172.21.15.40:0/4223734334 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69db7fe700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6e64d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.675 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69daffd700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e36e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 -- 172.21.15.40:0/4223734334 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 msgr2=0x562f0e6e64d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6e64d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 -- 172.21.15.40:0/4223734334 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 msgr2=0x562f0e6e36e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e36e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 -- 172.21.15.40:0/4223734334 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69d0019040 con 0x562f0e6e6ad0 2023-07-14T11:14:13.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69daffd700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e36e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:13.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69dbfff700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f69d40a1db0 tx=0x7f69d409dec0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1a6469b6031f0d55 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69baffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:13.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69d409a2f0 con 0x562f0e6e6ad0 2023-07-14T11:14:13.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.670+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f69d40128e0 con 0x562f0e6e6ad0 2023-07-14T11:14:13.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562f0e5874f0 con 0x562f0e6e6ad0 2023-07-14T11:14:13.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f69d400ae10 con 0x562f0e6e6ad0 2023-07-14T11:14:13.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 10 client.26132 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:13.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x562f0e444c60 con 0x562f0e6e6ad0 2023-07-14T11:14:13.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69d40b8030 con 0x562f0e6e6ad0 2023-07-14T11:14:13.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f69d40cab50 con 0x562f0e6e6ad0 2023-07-14T11:14:13.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 10 client.26132 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:13.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 10 client.26132 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:14:13.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.671+0000 7f69f5a75b80 20 client.26132 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:13.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.672+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e6f63e0 0x562f0e6f8830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.672+0000 7f69f5a75b80 4 client.26132 mds_command: new command op to 24521 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:13.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.672+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562f0e626eb0 con 0x562f0e6f63e0 2023-07-14T11:14:13.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.673+0000 7f69db7fe700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6e64d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:13.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.673+0000 7f69db7fe700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e6f63e0 0x562f0e6f8830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.675+0000 7f69db7fe700 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e6f63e0 0x562f0e6f8830 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=5d5a18219665c62f server_cookie=a261c5d9f88472c5 in_seq=0 out_seq=0 2023-07-14T11:14:13.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.675+0000 7f69baffd700 10 client.26132 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:14:13.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69baffd700 1 -- 172.21.15.40:0/4223734334 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562f0e626eb0 con 0x562f0e6f63e0 2023-07-14T11:14:13.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69baffd700 10 client.26132 handle_command_reply: tid=0 2023-07-14T11:14:13.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 client.26132 shutdown 2023-07-14T11:14:13.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 msgr2=0x562f0e6e3cf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f69d40a1db0 tx=0x7f69d409dec0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 shutdown_connections 2023-07-14T11:14:13.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e6f63e0 0x562f0e6f8830 crc :-1 s=READY pgs=48 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e3cf0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6e64d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/4223734334 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e36e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 >> 172.21.15.40:0/4223734334 conn(0x562f0e6d1da0 msgr2=0x562f0e6d7230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.676+0000 7f69baffd700 0 client.26132 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:14:13.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.677+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 shutdown_connections 2023-07-14T11:14:13.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.677+0000 7f69f5a75b80 1 -- 172.21.15.40:0/4223734334 wait complete. 2023-07-14T11:14:13.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.677+0000 7f69f5a75b80 20 client.26132 trim_cache size 0 max 16384 2023-07-14T11:14:13.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.698+0000 7f69f5a75b80 1 Processor -- start 2023-07-14T11:14:13.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.698+0000 7f69f5a75b80 1 -- start start 2023-07-14T11:14:13.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e09c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6d1970 con 0x562f0e6e6ad0 2023-07-14T11:14:13.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6dcde0 con 0x562f0e6db2d0 2023-07-14T11:14:13.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6df110 con 0x562f0e6dbbe0 2023-07-14T11:14:13.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34666/0 (socket says 172.21.15.40:34666) 2023-07-14T11:14:13.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 -- 172.21.15.40:0/2092109188 learned_addr learned my addr 172.21.15.40:0/2092109188 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69daffd700 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69dbfff700 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e09c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 -- 172.21.15.40:0/2092109188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 msgr2=0x562f0e6e6000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 -- 172.21.15.40:0/2092109188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 msgr2=0x562f0e6e09c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.699+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e09c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.700+0000 7f69db7fe700 1 -- 172.21.15.40:0/2092109188 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562f0e482250 con 0x562f0e6db2d0 2023-07-14T11:14:13.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.700+0000 7f69db7fe700 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f69cc00b6f0 tx=0x7f69cc021000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fc3a3928b41855b6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.700+0000 7f69c27fc700 1 -- 172.21.15.40:0/2092109188 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69cc033d30 con 0x562f0e6db2d0 2023-07-14T11:14:13.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.700+0000 7f69c27fc700 1 -- 172.21.15.40:0/2092109188 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69cc02b040 con 0x562f0e6db2d0 2023-07-14T11:14:13.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.700+0000 7f69c27fc700 1 -- 172.21.15.40:0/2092109188 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69cc00f460 con 0x562f0e6db2d0 2023-07-14T11:14:13.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 -- 172.21.15.40:0/2092109188 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 msgr2=0x562f0e6ed4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f69cc00b6f0 tx=0x7f69cc021000 comp rx=0 tx=0).stop 2023-07-14T11:14:13.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 -- 172.21.15.40:0/2092109188 shutdown_connections 2023-07-14T11:14:13.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6e09c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed4e0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/2092109188 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6dbbe0 0x562f0e6e6000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 -- 172.21.15.40:0/2092109188 >> 172.21.15.40:0/2092109188 conn(0x562f0e6d1da0 msgr2=0x562f0e6f83f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 -- 172.21.15.40:0/2092109188 shutdown_connections 2023-07-14T11:14:13.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.701+0000 7f69f5a75b80 1 -- 172.21.15.40:0/2092109188 wait complete. 2023-07-14T11:14:13.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.702+0000 7f69f5a75b80 1 -- start start 2023-07-14T11:14:13.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.702+0000 7f69f5a75b80 1 Processor -- start 2023-07-14T11:14:13.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6dbbe0 0x562f0e6ed690 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6eb610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6ebd20 con 0x562f0e6e6ad0 2023-07-14T11:14:13.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6ebe90 con 0x562f0e6dbbe0 2023-07-14T11:14:13.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562f0e6ec050 con 0x562f0e6db2d0 2023-07-14T11:14:13.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69f5a75b80 10 client.0 _ll_register_callbacks cb 0x7f69d4099a60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:13.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69db7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60610/0 (socket says 172.21.15.40:60610) 2023-07-14T11:14:13.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69db7fe700 1 -- 172.21.15.40:0/3767267856 learned_addr learned my addr 172.21.15.40:0/3767267856 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.703+0000 7f69daffd700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6dbbe0 0x562f0e6ed690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69dbfff700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6eb610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 -- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6dbbe0 msgr2=0x562f0e6ed690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6dbbe0 0x562f0e6ed690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 -- 172.21.15.40:0/3767267856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 msgr2=0x562f0e6eb610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6eb610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 -- 172.21.15.40:0/3767267856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69dc10d170 con 0x562f0e6db2d0 2023-07-14T11:14:13.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69dbfff700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6eb610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:13.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69db7fe700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f69cc000c00 tx=0x7f69cc005b00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=be9a50e4900361e6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69baffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:13.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69cc025bd0 con 0x562f0e6db2d0 2023-07-14T11:14:13.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f69d0017070 con 0x562f0e6db2d0 2023-07-14T11:14:13.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.704+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f69cc025d30 con 0x562f0e6db2d0 2023-07-14T11:14:13.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f69cc033040 con 0x562f0e6db2d0 2023-07-14T11:14:13.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 10 client.26104 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:13.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x562f0e45a2b0 con 0x562f0e6db2d0 2023-07-14T11:14:13.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69d0017070 con 0x562f0e6db2d0 2023-07-14T11:14:13.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f69cc00f910 con 0x562f0e6db2d0 2023-07-14T11:14:13.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 10 client.26104 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:13.722 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 10 client.26104 resolve_mds: resolved 1:3 to role '1:3' aka daemon mds.h 2023-07-14T11:14:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 20 client.26104 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e74b270 0x562f0e74d680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 4 client.26104 mds_command: new command op to 24521 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:14:13.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.705+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 --> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x562f0e5e9050 con 0x562f0e74b270 2023-07-14T11:14:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69daffd700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e74b270 0x562f0e74d680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69daffd700 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e74b270 0x562f0e74d680 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.3 client_cookie=1cc74da84e5a0b61 server_cookie=823638a0e2a8d100 in_seq=0 out_seq=0 2023-07-14T11:14:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69baffd700 10 client.26104 ms_handle_connect on v2:172.21.15.45:6836/3949707739 2023-07-14T11:14:13.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69baffd700 1 -- 172.21.15.40:0/3767267856 <== mds.3 v2:172.21.15.45:6836/3949707739 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x562f0e5e9050 con 0x562f0e74b270 2023-07-14T11:14:13.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69baffd700 10 client.26104 handle_command_reply: tid=0 2023-07-14T11:14:13.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.706+0000 7f69f5a75b80 1 client.26104 shutdown 2023-07-14T11:14:13.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 msgr2=0x562f0e6ed150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7f69cc000c00 tx=0x7f69cc005b00 comp rx=0 tx=0).stop 2023-07-14T11:14:13.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 shutdown_connections 2023-07-14T11:14:13.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:6836/3949707739,v1:172.21.15.45:6837/3949707739] conn(0x562f0e74b270 0x562f0e74d680 crc :-1 s=READY pgs=49 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562f0e6e6ad0 0x562f0e6eb610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562f0e6db2d0 0x562f0e6ed150 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 --2- 172.21.15.40:0/3767267856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562f0e6dbbe0 0x562f0e6ed690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69baffd700 0 client.26104 ms_handle_reset on v2:172.21.15.45:6836/3949707739 2023-07-14T11:14:13.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 >> 172.21.15.40:0/3767267856 conn(0x562f0e6d1da0 msgr2=0x562f0e6e1ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 shutdown_connections 2023-07-14T11:14:13.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 1 -- 172.21.15.40:0/3767267856 wait complete. 2023-07-14T11:14:13.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.707+0000 7f69f5a75b80 20 client.26104 trim_cache size 0 max 16384 2023-07-14T11:14:13.729 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:14:13.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 msgr2=0x7f69c40a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 0x7f69c40a37f0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f69cc006010 tx=0x7f69cc01b040 comp rx=0 tx=0).stop 2023-07-14T11:14:13.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 msgr2=0x7f69dc19c7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 0x7f69dc19c7c0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f69d400cc50 tx=0x7f69d400cf60 comp rx=0 tx=0).stop 2023-07-14T11:14:13.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 shutdown_connections 2023-07-14T11:14:13.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69c40a1340 0x7f69c40a37f0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69dc10a070 0x7f69dc19c7c0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69dc100ca0 0x7f69dc197e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 --2- 172.21.15.40:0/2948460134 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69dc1015b0 0x7f69dc198360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 >> 172.21.15.40:0/2948460134 conn(0x7f69dc0fa4b0 msgr2=0x7f69dc0fa880 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 shutdown_connections 2023-07-14T11:14:13.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.708+0000 7f69e21ff700 1 -- 172.21.15.40:0/2948460134 wait complete. 2023-07-14T11:14:13.734 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:2 damage ls 2023-07-14T11:14:13.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 -- 172.21.15.40:0/3258555201 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 msgr2=0x7f7248072c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 --2- 172.21.15.40:0/3258555201 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248072c20 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f7230009990 tx=0x7f7230009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 -- 172.21.15.40:0/3258555201 shutdown_connections 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 --2- 172.21.15.40:0/3258555201 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 0x7f724810e780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 --2- 172.21.15.40:0/3258555201 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248072c20 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.978+0000 7f724dfff700 1 --2- 172.21.15.40:0/3258555201 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f724810be60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.979+0000 7f724dfff700 1 -- 172.21.15.40:0/3258555201 >> 172.21.15.40:0/3258555201 conn(0x7f72480fb600 msgr2=0x7f72480fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.979+0000 7f724dfff700 1 -- 172.21.15.40:0/3258555201 shutdown_connections 2023-07-14T11:14:13.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.979+0000 7f724dfff700 1 -- 172.21.15.40:0/3258555201 wait complete. 2023-07-14T11:14:13.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.980+0000 7f724dfff700 1 Processor -- start 2023-07-14T11:14:13.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.980+0000 7f724dfff700 1 -- start start 2023-07-14T11:14:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248197e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 0x7f724819c820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819ce60 con 0x7f724810c3a0 2023-07-14T11:14:13.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819cfd0 con 0x7f7248072850 2023-07-14T11:14:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f724dfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f724819d190 con 0x7f7248073160 2023-07-14T11:14:13.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f72477fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248197e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60634/0 (socket says 172.21.15.40:60634) 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.981+0000 7f7246ffd700 1 -- 172.21.15.40:0/3118621079 learned_addr learned my addr 172.21.15.40:0/3118621079 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f7247fff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 0x7f724819c820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 -- 172.21.15.40:0/3118621079 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 msgr2=0x7f72481983c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 -- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 msgr2=0x7f724819c820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 0x7f724819c820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 -- 172.21.15.40:0/3118621079 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72300095f0 con 0x7f7248072850 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f7246ffd700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f72477fe700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248197e80 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f72300098e0 tx=0x7f7230004730 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.982+0000 7f7244ff9700 1 -- 172.21.15.40:0/3118621079 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723001d070 con 0x7f7248072850 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.983+0000 7f7244ff9700 1 -- 172.21.15.40:0/3118621079 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f72300054f0 con 0x7f7248072850 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.983+0000 7f7244ff9700 1 -- 172.21.15.40:0/3118621079 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7230004c00 con 0x7f7248072850 2023-07-14T11:14:13.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.983+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f724819d410 con 0x7f7248072850 2023-07-14T11:14:13.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.983+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72481a4ad0 con 0x7f7248072850 2023-07-14T11:14:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.987+0000 7f72619b2b80 1 Processor -- start 2023-07-14T11:14:13.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 -- start start 2023-07-14T11:14:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b552a6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b553dda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b55406c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b5430cb0 con 0x55f4b552abe0 2023-07-14T11:14:13.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b5483a40 con 0x55f4b552a2d0 2023-07-14T11:14:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.988+0000 7f72619b2b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b547ced0 con 0x55f4b553e2e0 2023-07-14T11:14:13.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7244ff9700 1 -- 172.21.15.40:0/3118621079 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f723000fb90 con 0x7f7248072850 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b55406c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b552a6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b55406c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60644/0 (socket says 172.21.15.40:60644) 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7246ffd700 1 -- 172.21.15.40:0/4033913061 learned_addr learned my addr 172.21.15.40:0/4033913061 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f72477fe700 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b553dda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 -- 172.21.15.40:0/4033913061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 msgr2=0x55f4b55406c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b55406c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 -- 172.21.15.40:0/4033913061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 msgr2=0x55f4b553dda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b553dda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f7247fff700 1 -- 172.21.15.40:0/4033913061 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f4b52aade0 con 0x55f4b552a2d0 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.989+0000 7f72477fe700 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b553dda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.990+0000 7f7247fff700 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b552a6a0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7248074920 tx=0x7f7240009f40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c35eda8c0344295c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.990+0000 7f723dffb700 1 -- 172.21.15.40:0/4033913061 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7240018070 con 0x55f4b552a2d0 2023-07-14T11:14:13.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.990+0000 7f723dffb700 1 -- 172.21.15.40:0/4033913061 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f724000f050 con 0x55f4b552a2d0 2023-07-14T11:14:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.990+0000 7f723dffb700 1 -- 172.21.15.40:0/4033913061 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7240005940 con 0x55f4b552a2d0 2023-07-14T11:14:13.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f7244ff9700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 0x7f72340a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 -- 172.21.15.40:0/4033913061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 msgr2=0x55f4b552a6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b552a6a0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f7248074920 tx=0x7f7240009f40 comp rx=0 tx=0).stop 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f7244ff9700 1 -- 172.21.15.40:0/3118621079 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f72300ce130 con 0x7f7248072850 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 -- 172.21.15.40:0/4033913061 shutdown_connections 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b55406c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b552a6a0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4033913061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b553dda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72619b2b80 1 -- 172.21.15.40:0/4033913061 >> 172.21.15.40:0/4033913061 conn(0x55f4b5520da0 msgr2=0x55f4b55266a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.991+0000 7f72477fe700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 0x7f72340a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.992+0000 7f72619b2b80 1 -- 172.21.15.40:0/4033913061 shutdown_connections 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.992+0000 7f72619b2b80 1 -- 172.21.15.40:0/4033913061 wait complete. 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.992+0000 7f72619b2b80 1 -- start start 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.992+0000 7f72619b2b80 1 Processor -- start 2023-07-14T11:14:13.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72477fe700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 0x7f72340a37f0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f723009ac20 tx=0x7f7230009ca0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b553afc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553b700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b541cac0 con 0x55f4b553e2e0 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b541c7c0 con 0x55f4b552a2d0 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b5300920 con 0x55f4b552abe0 2023-07-14T11:14:13.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f72619b2b80 10 client.0 _ll_register_callbacks cb 0x55f4b541a760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:13.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f7247fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f7247fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34704/0 (socket says 172.21.15.40:34704) 2023-07-14T11:14:13.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.993+0000 7f7247fff700 1 -- 172.21.15.40:0/2647444840 learned_addr learned my addr 172.21.15.40:0/2647444840 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:13.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553b700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:13.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7246ffd700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b553afc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 -- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 msgr2=0x55f4b553afc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b553afc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 -- 172.21.15.40:0/2647444840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 msgr2=0x55f4b553b700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553b700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7247fff700 1 -- 172.21.15.40:0/2647444840 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72400098c0 con 0x55f4b552a2d0 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.994+0000 7f7246ffd700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b553afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:14.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7247fff700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f7240006090 tx=0x7f7240004f00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fe4dbbfbc9ae21d5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7226ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7240018040 con 0x55f4b552a2d0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55f4b53d64f0 con 0x55f4b552a2d0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f724001f660 con 0x55f4b552a2d0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f724001f820 con 0x55f4b552a2d0 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f72619b2b80 10 client.26150 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:14.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x55f4b5293c60 con 0x55f4b552a2d0 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.995+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55f4b53d64f0 con 0x55f4b552a2d0 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f72400217b0 con 0x55f4b552a2d0 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 10 client.26150 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 10 client.26150 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 20 client.26150 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b554da10 0x55f4b554fe20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 4 client.26150 mds_command: new command op to 24523 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:14.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55f4b5475eb0 con 0x55f4b554da10 2023-07-14T11:14:14.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.996+0000 7f7246ffd700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b554da10 0x55f4b554fe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.997+0000 7f7246ffd700 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b554da10 0x55f4b554fe20 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=e2d4e3cf022081a5 server_cookie=b4c01ce70e55565e in_seq=0 out_seq=0 2023-07-14T11:14:14.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.997+0000 7f7226ffd700 10 client.26150 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f7226ffd700 1 -- 172.21.15.40:0/2647444840 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55f4b5475eb0 con 0x55f4b554da10 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f7226ffd700 10 client.26150 handle_command_reply: tid=0 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 client.26150 shutdown 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 msgr2=0x55f4b553dd40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f7240006090 tx=0x7f7240004f00 comp rx=0 tx=0).stop 2023-07-14T11:14:14.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 shutdown_connections 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b554da10 0x55f4b554fe20 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553b700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553dd40 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 --2- 172.21.15.40:0/2647444840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b553afc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 >> 172.21.15.40:0/2647444840 conn(0x55f4b5520da0 msgr2=0x55f4b5526230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f7226ffd700 0 client.26150 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:14:14.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 shutdown_connections 2023-07-14T11:14:14.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.998+0000 7f72619b2b80 1 -- 172.21.15.40:0/2647444840 wait complete. 2023-07-14T11:14:14.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:13.999+0000 7f72619b2b80 20 client.26150 trim_cache size 0 max 16384 2023-07-14T11:14:14.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.013+0000 7f72619b2b80 1 Processor -- start 2023-07-14T11:14:14.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.013+0000 7f72619b2b80 1 -- start start 2023-07-14T11:14:14.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553d870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b552b540 con 0x55f4b553e2e0 2023-07-14T11:14:14.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b552d1b0 con 0x55f4b552a2d0 2023-07-14T11:14:14.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f72619b2b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b552a8f0 con 0x55f4b552abe0 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f7246ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60656/0 (socket says 172.21.15.40:60656) 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f7246ffd700 1 -- 172.21.15.40:0/1596232503 learned_addr learned my addr 172.21.15.40:0/1596232503 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f7247fff700 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.014+0000 7f7247fff700 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553d870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 -- 172.21.15.40:0/1596232503 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 msgr2=0x55f4b553b6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553b6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 -- 172.21.15.40:0/1596232503 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 msgr2=0x55f4b553d870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553d870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 -- 172.21.15.40:0/1596232503 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55f4b52d1250 con 0x55f4b552abe0 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f7246ffd700 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f7238004290 tx=0x7f7238009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=86faaa8649ba32bb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.015+0000 7f723e7fc700 1 -- 172.21.15.40:0/1596232503 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f723801e330 con 0x55f4b552abe0 2023-07-14T11:14:14.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.016+0000 7f723e7fc700 1 -- 172.21.15.40:0/1596232503 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7238017040 con 0x55f4b552abe0 2023-07-14T11:14:14.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.016+0000 7f723e7fc700 1 -- 172.21.15.40:0/1596232503 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72380108d0 con 0x55f4b552abe0 2023-07-14T11:14:14.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- 172.21.15.40:0/1596232503 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 msgr2=0x55f4b5549460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7f7238004290 tx=0x7f7238009260 comp rx=0 tx=0).stop 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- 172.21.15.40:0/1596232503 shutdown_connections 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b553e2e0 0x55f4b553d870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b553b6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 --2- 172.21.15.40:0/1596232503 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b552abe0 0x55f4b5549460 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- 172.21.15.40:0/1596232503 >> 172.21.15.40:0/1596232503 conn(0x55f4b5520da0 msgr2=0x55f4b5549f10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- 172.21.15.40:0/1596232503 shutdown_connections 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- 172.21.15.40:0/1596232503 wait complete. 2023-07-14T11:14:14.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.017+0000 7f72619b2b80 1 -- start start 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.018+0000 7f72619b2b80 1 Processor -- start 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.018+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.018+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b5542ce0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f72619b2b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b5543220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f72619b2b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b55452a0 con 0x55f4b552abe0 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f72619b2b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b5543760 con 0x55f4b552a2d0 2023-07-14T11:14:14.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f72619b2b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55f4b55438f0 con 0x55f4b553e2e0 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f72619b2b80 10 client.0 _ll_register_callbacks cb 0x7f72400052f0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f7247fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f7247fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34710/0 (socket says 172.21.15.40:34710) 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f7247fff700 1 -- 172.21.15.40:0/4186460826 learned_addr learned my addr 172.21.15.40:0/4186460826 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f7246ffd700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b5542ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.019+0000 7f7247fff700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b5543220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 -- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 msgr2=0x55f4b5543220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b5543220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 -- 172.21.15.40:0/4186460826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 msgr2=0x55f4b5542ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b5542ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 -- 172.21.15.40:0/4186460826 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72380044c0 con 0x55f4b552a2d0 2023-07-14T11:14:14.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7247fff700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f7240009bb0 tx=0x7f7240004bb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1fe1f622d8457179 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7226ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f72400077e0 con 0x55f4b552a2d0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7240007cc0 con 0x55f4b552a2d0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.020+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f724000f050 con 0x55f4b552a2d0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.021+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7240021660 con 0x55f4b552a2d0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.021+0000 7f72619b2b80 10 client.26156 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.021+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x55f4b52a92b0 con 0x55f4b552a2d0 2023-07-14T11:14:14.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.021+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f724000f050 con 0x55f4b552a2d0 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f724003c060 con 0x55f4b552a2d0 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 10 client.26156 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 10 client.26156 resolve_mds: resolved 1:2 to role '1:2' aka daemon mds.c 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 20 client.26156 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b55a26c0 0x55f4b55a4aa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 4 client.26156 mds_command: new command op to 24523 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.022+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 --> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55f4b5438050 con 0x55f4b55a26c0 2023-07-14T11:14:14.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.023+0000 7f7246ffd700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b55a26c0 0x55f4b55a4aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.023+0000 7f7246ffd700 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b55a26c0 0x55f4b55a4aa0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.2 client_cookie=cfd5622944ed022c server_cookie=9ed6db9d5c7f8280 in_seq=0 out_seq=0 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.023+0000 7f7226ffd700 10 client.26156 ms_handle_connect on v2:172.21.15.130:6836/4266211241 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f7226ffd700 1 -- 172.21.15.40:0/4186460826 <== mds.2 v2:172.21.15.130:6836/4266211241 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55f4b5438050 con 0x55f4b55a26c0 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f7226ffd700 10 client.26156 handle_command_reply: tid=0 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 client.26156 shutdown 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 msgr2=0x55f4b55449d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f7240009bb0 tx=0x7f7240004bb0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 shutdown_connections 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:6836/4266211241,v1:172.21.15.130:6837/4266211241] conn(0x55f4b55a26c0 0x55f4b55a4aa0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55f4b553e2e0 0x55f4b5543220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55f4b552a2d0 0x55f4b55449d0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 --2- 172.21.15.40:0/4186460826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55f4b552abe0 0x55f4b5542ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 >> 172.21.15.40:0/4186460826 conn(0x55f4b5520da0 msgr2=0x55f4b552c1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f7226ffd700 0 client.26156 ms_handle_reset on v2:172.21.15.130:6836/4266211241 2023-07-14T11:14:14.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 shutdown_connections 2023-07-14T11:14:14.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 1 -- 172.21.15.40:0/4186460826 wait complete. 2023-07-14T11:14:14.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.024+0000 7f72619b2b80 20 client.26156 trim_cache size 0 max 16384 2023-07-14T11:14:14.030 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:14:14.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 msgr2=0x7f72340a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 0x7f72340a37f0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f723009ac20 tx=0x7f7230009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 msgr2=0x7f7248197e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248197e80 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f72300098e0 tx=0x7f7230004730 comp rx=0 tx=0).stop 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 shutdown_connections 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72340a1340 0x7f72340a37f0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f724810c3a0 0x7f724819c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7248072850 0x7f7248197e80 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 --2- 172.21.15.40:0/3118621079 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7248073160 0x7f72481983c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.025+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 >> 172.21.15.40:0/3118621079 conn(0x7f72480fb600 msgr2=0x7f724810e4a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.026+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 shutdown_connections 2023-07-14T11:14:14.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.026+0000 7f724dfff700 1 -- 172.21.15.40:0/3118621079 wait complete. 2023-07-14T11:14:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:13 smithi130 ceph-mon[129405]: pgmap v65: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 1.8 MiB/s wr, 233 op/s 2023-07-14T11:14:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:13 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3484695434' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:14:14.054 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:1 damage ls 2023-07-14T11:14:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:13 smithi045 ceph-mon[128791]: pgmap v65: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 1.8 MiB/s wr, 233 op/s 2023-07-14T11:14:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:13 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3484695434' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:14:14.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 -- 172.21.15.40:0/15726357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f01015b0 msgr2=0x7f48f0109aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 --2- 172.21.15.40:0/15726357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f01015b0 0x7f48f0109aa0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7f48e4009990 tx=0x7f48e4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 -- 172.21.15.40:0/15726357 shutdown_connections 2023-07-14T11:14:14.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 --2- 172.21.15.40:0/15726357 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f010a070 0x7f48f010c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 --2- 172.21.15.40:0/15726357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f0100ca0 0x7f48f0101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 --2- 172.21.15.40:0/15726357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f01015b0 0x7f48f0109aa0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.276+0000 7f48f55ff700 1 -- 172.21.15.40:0/15726357 >> 172.21.15.40:0/15726357 conn(0x7f48f00fa4b0 msgr2=0x7f48f00fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.277+0000 7f48f55ff700 1 -- 172.21.15.40:0/15726357 shutdown_connections 2023-07-14T11:14:14.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.277+0000 7f48f55ff700 1 -- 172.21.15.40:0/15726357 wait complete. 2023-07-14T11:14:14.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 Processor -- start 2023-07-14T11:14:14.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 -- start start 2023-07-14T11:14:14.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 0x7f48f01a3e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f00ff660 con 0x7f48f010a070 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f01a4350 con 0x7f48f01015b0 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48f55ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48f01a4490 con 0x7f48f0100ca0 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34720/0 (socket says 172.21.15.40:34720) 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.278+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60678/0 (socket says 172.21.15.40:60678) 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2678966994 learned_addr learned my addr 172.21.15.40:0/2678966994 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 0x7f48f01a3e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2678966994 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 msgr2=0x7f48f00fe8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 msgr2=0x7f48f01a3e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 0x7f48f01a3e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2678966994 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f48e40095f0 con 0x7f48f01015b0 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 0x7f48f01a3e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48eeffd700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.279+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f48e4005910 tx=0x7f48e4005720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.280+0000 7f48d7fff700 1 -- 172.21.15.40:0/2678966994 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e401d070 con 0x7f48f01015b0 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.280+0000 7f48d7fff700 1 -- 172.21.15.40:0/2678966994 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f48e4022470 con 0x7f48f01015b0 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.280+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48f01a4690 con 0x7f48f01015b0 2023-07-14T11:14:14.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.280+0000 7f48d7fff700 1 -- 172.21.15.40:0/2678966994 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e4017650 con 0x7f48f01015b0 2023-07-14T11:14:14.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.280+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48f01a4b60 con 0x7f48f01015b0 2023-07-14T11:14:14.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.283+0000 7f48d7fff700 1 -- 172.21.15.40:0/2678966994 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f48e40225e0 con 0x7f48f01015b0 2023-07-14T11:14:14.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.284+0000 7f490901db80 1 Processor -- start 2023-07-14T11:14:14.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 -- start start 2023-07-14T11:14:14.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7dfc6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e09eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7d02cb0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7d55a40 con 0x560ae7dfcbe0 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f490901db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7d4eed0 con 0x560ae7e07ad0 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e09eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34726/0 (socket says 172.21.15.40:34726) 2023-07-14T11:14:14.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f48eeffd700 1 -- 172.21.15.40:0/2511020109 learned_addr learned my addr 172.21.15.40:0/2511020109 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.285+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7dfc6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48d7fff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 0x7f48d80a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 -- 172.21.15.40:0/2511020109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 msgr2=0x560ae7e09eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e09eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 -- 172.21.15.40:0/2511020109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 msgr2=0x560ae7dfc6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7dfc6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 -- 172.21.15.40:0/2511020109 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ae7b7cde0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48d7fff700 1 -- 172.21.15.40:0/2678966994 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f48e40ce3b0 con 0x7f48f01015b0 2023-07-14T11:14:14.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 0x7f48d80a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.286+0000 7f48eeffd700 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f48e00115f0 tx=0x7f48e0011900 comp rx=0 tx=0).ready entity=mon.2 client_cookie=13eeaa0e6402e9ca server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.287+0000 7f48d57fa700 1 -- 172.21.15.40:0/2511020109 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e0019630 con 0x560ae7dfcbe0 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.287+0000 7f48d57fa700 1 -- 172.21.15.40:0/2511020109 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f48e0019c50 con 0x560ae7dfcbe0 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.287+0000 7f48d57fa700 1 -- 172.21.15.40:0/2511020109 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e000ec20 con 0x560ae7dfcbe0 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 -- 172.21.15.40:0/2511020109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 msgr2=0x560ae7e07590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 0x7f48d80a37f0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f48e80094f0 tx=0x7f48e8009040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.293 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f48e00115f0 tx=0x7f48e0011900 comp rx=0 tx=0).stop 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 -- 172.21.15.40:0/2511020109 shutdown_connections 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e09eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7dfc6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 --2- 172.21.15.40:0/2511020109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07590 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 -- 172.21.15.40:0/2511020109 >> 172.21.15.40:0/2511020109 conn(0x560ae7df2da0 msgr2=0x560ae7df86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 -- 172.21.15.40:0/2511020109 shutdown_connections 2023-07-14T11:14:14.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.288+0000 7f490901db80 1 -- 172.21.15.40:0/2511020109 wait complete. 2023-07-14T11:14:14.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 -- start start 2023-07-14T11:14:14.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 Processor -- start 2023-07-14T11:14:14.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e074b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.295 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e04e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7ceeac0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7cee7c0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.289+0000 7f490901db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7bd2920 con 0x560ae7e07ad0 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f490901db80 10 client.0 _ll_register_callbacks cb 0x560ae7cec760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ef7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e04e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34740/0 (socket says 172.21.15.40:34740) 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 -- 172.21.15.40:0/450789464 learned_addr learned my addr 172.21.15.40:0/450789464 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48eeffd700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e074b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 -- 172.21.15.40:0/450789464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 msgr2=0x560ae7e04e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e04e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 -- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 msgr2=0x560ae7e074b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e074b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 -- 172.21.15.40:0/450789464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48e0009dd0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48eeffd700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e074b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f48e400f740 tx=0x7f48e40cbcf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1ed733bc09aecb47 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.290+0000 7f48d2ff5700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:14.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e40974e0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f48e409b1a0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x560ae7ca84f0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f48e409af70 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 10 client.26168 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x560ae7b65c60 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x560ae7ca84f0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f48e409a690 con 0x560ae7dfcbe0 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 10 client.26168 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 10 client.26168 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 20 client.26168 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e17020 0x560ae7e19430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 4 client.26168 mds_command: new command op to 24565 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:14.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.291+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x560ae7d47eb0 con 0x560ae7e17020 2023-07-14T11:14:14.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.292+0000 7f48eeffd700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e17020 0x560ae7e19430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.292+0000 7f48eeffd700 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e17020 0x560ae7e19430 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=ff58d45c445c904c server_cookie=809633ef08009f87 in_seq=0 out_seq=0 2023-07-14T11:14:14.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.292+0000 7f48d2ff5700 10 client.26168 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:14:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f48d2ff5700 1 -- 172.21.15.40:0/450789464 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x560ae7d47eb0 con 0x560ae7e17020 2023-07-14T11:14:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f48d2ff5700 10 client.26168 handle_command_reply: tid=0 2023-07-14T11:14:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f490901db80 1 client.26168 shutdown 2023-07-14T11:14:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 msgr2=0x560ae7e047a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f48e400f740 tx=0x7f48e40cbcf0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 shutdown_connections 2023-07-14T11:14:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.481+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e17020 0x560ae7e19430 crc :-1 s=READY pgs=44 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e04e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f48d2ff5700 0 client.26168 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e074b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 --2- 172.21.15.40:0/450789464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e047a0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 >> 172.21.15.40:0/450789464 conn(0x560ae7df2da0 msgr2=0x560ae7df8290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 shutdown_connections 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 1 -- 172.21.15.40:0/450789464 wait complete. 2023-07-14T11:14:14.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.482+0000 7f490901db80 20 client.26168 trim_cache size 0 max 16384 2023-07-14T11:14:14.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.510+0000 7f490901db80 1 Processor -- start 2023-07-14T11:14:14.512 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 -- start start 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e0e4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e01a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7deffa0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7dff250 con 0x560ae7dfcbe0 2023-07-14T11:14:14.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f490901db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7df2970 con 0x560ae7e07ad0 2023-07-14T11:14:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f48ee7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34756/0 (socket says 172.21.15.40:34756) 2023-07-14T11:14:14.514 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.511+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2979415459 learned_addr learned my addr 172.21.15.40:0/2979415459 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2979415459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 msgr2=0x560ae7e01a80 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e01a80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2979415459 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 msgr2=0x560ae7e0e4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e0e4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 -- 172.21.15.40:0/2979415459 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x560ae7ba3250 con 0x560ae7dfcbe0 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f48e40a3030 tx=0x7f48e4022c90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c35e17c01f5d7663 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.515 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48d5ffb700 1 -- 172.21.15.40:0/2979415459 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e40974e0 con 0x560ae7dfcbe0 2023-07-14T11:14:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.512+0000 7f48d5ffb700 1 -- 172.21.15.40:0/2979415459 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f48e409af70 con 0x560ae7dfcbe0 2023-07-14T11:14:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.513+0000 7f48d5ffb700 1 -- 172.21.15.40:0/2979415459 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e409a690 con 0x560ae7dfcbe0 2023-07-14T11:14:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.513+0000 7f490901db80 1 -- 172.21.15.40:0/2979415459 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 msgr2=0x560ae7e07050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.516 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.513+0000 7f490901db80 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f48e40a3030 tx=0x7f48e4022c90 comp rx=0 tx=0).stop 2023-07-14T11:14:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 -- 172.21.15.40:0/2979415459 shutdown_connections 2023-07-14T11:14:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7e07ad0 0x560ae7e01a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7dfc2d0 0x560ae7e0e4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.517 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 --2- 172.21.15.40:0/2979415459 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e07050 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 -- 172.21.15.40:0/2979415459 >> 172.21.15.40:0/2979415459 conn(0x560ae7df2da0 msgr2=0x560ae7dffaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 -- 172.21.15.40:0/2979415459 shutdown_connections 2023-07-14T11:14:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 -- 172.21.15.40:0/2979415459 wait complete. 2023-07-14T11:14:14.518 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.515+0000 7f490901db80 1 -- start start 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 Processor -- start 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e637c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7e07ad0 0x560ae7e5ee30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7e5f400 con 0x560ae7e07ad0 2023-07-14T11:14:14.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7e5f570 con 0x560ae7dfcbe0 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x560ae7e5f730 con 0x560ae7dfc2d0 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.516+0000 7f490901db80 10 client.0 _ll_register_callbacks cb 0x560ae7cec760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60726/0 (socket says 172.21.15.40:60726) 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 -- 172.21.15.40:0/584294441 learned_addr learned my addr 172.21.15.40:0/584294441 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e637c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48ef7fe700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7e07ad0 0x560ae7e5ee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 -- 172.21.15.40:0/584294441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 msgr2=0x560ae7e637c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e637c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 -- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7e07ad0 msgr2=0x560ae7e5ee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7e07ad0 0x560ae7e5ee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48eeffd700 1 -- 172.21.15.40:0/584294441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x560ae7b65c60 con 0x560ae7dfc2d0 2023-07-14T11:14:14.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.517+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e637c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:14.522 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f48eeffd700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f48e00115f0 tx=0x7f48e0009440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8c767b9dbb83afa6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f48d2ff5700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f48e002b520 con 0x560ae7dfc2d0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f48e0019dc0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f48e0019c50 con 0x560ae7dfc2d0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f48e002b7f0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.518+0000 7f490901db80 10 client.26137 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:14.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x560ae7b7b2b0 con 0x560ae7dfc2d0 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f48e0019c50 con 0x560ae7dfc2d0 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f48e0019960 con 0x560ae7dfc2d0 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 10 client.26137 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 10 client.26137 resolve_mds: resolved 1:1 to role '1:1' aka daemon mds.a 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 20 client.26137 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e6c2b0 0x560ae7e6e6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 4 client.26137 mds_command: new command op to 24565 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:14:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 --> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x560ae7d0a050 con 0x560ae7e6c2b0 2023-07-14T11:14:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.519+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e6c2b0 0x560ae7e6e6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.520+0000 7f48ee7fc700 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e6c2b0 0x560ae7e6e6c0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.1 client_cookie=42c6c28f03084c9 server_cookie=b54f8174183c8ec in_seq=0 out_seq=0 2023-07-14T11:14:14.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.520+0000 7f48d2ff5700 10 client.26137 ms_handle_connect on v2:172.21.15.40:6840/1989097261 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f48d2ff5700 1 -- 172.21.15.40:0/584294441 <== mds.1 v2:172.21.15.40:6840/1989097261 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x560ae7d0a050 con 0x560ae7e6c2b0 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f48d2ff5700 10 client.26137 handle_command_reply: tid=0 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 client.26137 shutdown 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 msgr2=0x560ae7e63280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7f48e00115f0 tx=0x7f48e0009440 comp rx=0 tx=0).stop 2023-07-14T11:14:14.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 shutdown_connections 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] conn(0x560ae7e6c2b0 0x560ae7e6e6c0 crc :-1 s=READY pgs=45 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x560ae7e07ad0 0x560ae7e5ee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x560ae7dfc2d0 0x560ae7e63280 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 --2- 172.21.15.40:0/584294441 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x560ae7dfcbe0 0x560ae7e637c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f48d2ff5700 0 client.26137 ms_handle_reset on v2:172.21.15.40:6840/1989097261 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 >> 172.21.15.40:0/584294441 conn(0x560ae7df2da0 msgr2=0x560ae7dfe1a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 shutdown_connections 2023-07-14T11:14:14.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.705+0000 7f490901db80 1 -- 172.21.15.40:0/584294441 wait complete. 2023-07-14T11:14:14.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f490901db80 20 client.26137 trim_cache size 0 max 16384 2023-07-14T11:14:14.709 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:14:14.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 msgr2=0x7f48d80a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 0x7f48d80a37f0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f48e80094f0 tx=0x7f48e8009040 comp rx=0 tx=0).stop 2023-07-14T11:14:14.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 msgr2=0x7f48f00fee30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f48e4005910 tx=0x7f48e4005720 comp rx=0 tx=0).stop 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 shutdown_connections 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f48d80a1340 0x7f48d80a37f0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f48f010a070 0x7f48f01a3e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f48f0100ca0 0x7f48f00fe8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 --2- 172.21.15.40:0/2678966994 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f48f01015b0 0x7f48f00fee30 secure :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f48e4005910 tx=0x7f48e4005720 comp rx=0 tx=0).stop 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.706+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 >> 172.21.15.40:0/2678966994 conn(0x7f48f00fa4b0 msgr2=0x7f48f010c1d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.707+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 shutdown_connections 2023-07-14T11:14:14.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.707+0000 7f48f55ff700 1 -- 172.21.15.40:0/2678966994 wait complete. 2023-07-14T11:14:14.735 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 damage ls 2023-07-14T11:14:14.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 -- 172.21.15.40:0/3796207518 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd97010a070 msgr2=0x7fd97010c4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 --2- 172.21.15.40:0/3796207518 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd97010a070 0x7fd97010c4b0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7fd968009f90 tx=0x7fd96800b330 comp rx=0 tx=0).stop 2023-07-14T11:14:14.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 -- 172.21.15.40:0/3796207518 shutdown_connections 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 --2- 172.21.15.40:0/3796207518 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd97010a070 0x7fd97010c4b0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 --2- 172.21.15.40:0/3796207518 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd970101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 --2- 172.21.15.40:0/3796207518 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd9701015b0 0x7fd970109aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 -- 172.21.15.40:0/3796207518 >> 172.21.15.40:0/3796207518 conn(0x7fd9700fa4b0 msgr2=0x7fd9700fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 -- 172.21.15.40:0/3796207518 shutdown_connections 2023-07-14T11:14:14.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.974+0000 7fd9749ff700 1 -- 172.21.15.40:0/3796207518 wait complete. 2023-07-14T11:14:14.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.976+0000 7fd9749ff700 1 Processor -- start 2023-07-14T11:14:14.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.976+0000 7fd9749ff700 1 -- start start 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd9701015b0 0x7fd9700fee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd97010a070 0x7fd9701a3ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9700ff660 con 0x7fd97010a070 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9701a4420 con 0x7fd9701015b0 2023-07-14T11:14:14.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd9749ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd9701a4560 con 0x7fd970100ca0 2023-07-14T11:14:14.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60744/0 (socket says 172.21.15.40:60744) 2023-07-14T11:14:14.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 -- 172.21.15.40:0/317243574 learned_addr learned my addr 172.21.15.40:0/317243574 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd9701015b0 0x7fd9700fee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd97010a070 0x7fd9701a3ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 -- 172.21.15.40:0/317243574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd9701015b0 msgr2=0x7fd9700fee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd9701015b0 0x7fd9700fee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 -- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd97010a070 msgr2=0x7fd9701a3ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd97010a070 0x7fd9701a3ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.977+0000 7fd96e59c700 1 -- 172.21.15.40:0/317243574 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd968009c40 con 0x7fd970100ca0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd96e59c700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fd96000e9c0 tx=0x7fd96000ecd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd95f7fe700 1 -- 172.21.15.40:0/317243574 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd96000cbd0 con 0x7fd970100ca0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd9701a47c0 con 0x7fd970100ca0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd95f7fe700 1 -- 172.21.15.40:0/317243574 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd96000eeb0 con 0x7fd970100ca0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd95f7fe700 1 -- 172.21.15.40:0/317243574 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd960005580 con 0x7fd970100ca0 2023-07-14T11:14:14.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.978+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd9701a4cf0 con 0x7fd970100ca0 2023-07-14T11:14:14.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.982+0000 7fd95f7fe700 1 -- 172.21.15.40:0/317243574 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fd960004750 con 0x7fd970100ca0 2023-07-14T11:14:14.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.983+0000 7fd95f7fe700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 0x7fd9580a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.984+0000 7fd95f7fe700 1 -- 172.21.15.40:0/317243574 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd960014070 con 0x7fd970100ca0 2023-07-14T11:14:14.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.984+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 0x7fd9580a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.985+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 0x7fd9580a37f0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd9701000e0 tx=0x7fd964009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.985+0000 7fd988418b80 1 Processor -- start 2023-07-14T11:14:14.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.985+0000 7fd988418b80 1 -- start start 2023-07-14T11:14:14.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 0x55565c72c590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c627cb0 con 0x55565c72cad0 2023-07-14T11:14:14.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c67aa40 con 0x55565c721be0 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd988418b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c673ed0 con 0x55565c7212d0 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60754/0 (socket says 172.21.15.40:60754) 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd96e59c700 1 -- 172.21.15.40:0/4177394928 learned_addr learned my addr 172.21.15.40:0/4177394928 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.986+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 0x55565c72c590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 -- 172.21.15.40:0/4177394928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 msgr2=0x55565c72c590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 0x55565c72c590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 -- 172.21.15.40:0/4177394928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 msgr2=0x55565c72eeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72eeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 -- 172.21.15.40:0/4177394928 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55565c4a1de0 con 0x55565c7212d0 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 0x55565c72c590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd96e59c700 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fd960096bc0 tx=0x7fd96000cdb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=fb0112489aae77b5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd95cff9700 1 -- 172.21.15.40:0/4177394928 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd96009e180 con 0x55565c7212d0 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd95cff9700 1 -- 172.21.15.40:0/4177394928 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd9600a25a0 con 0x55565c7212d0 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.987+0000 7fd95cff9700 1 -- 172.21.15.40:0/4177394928 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd96009a5d0 con 0x55565c7212d0 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.988+0000 7fd988418b80 1 -- 172.21.15.40:0/4177394928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 msgr2=0x55565c7216a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.988+0000 7fd988418b80 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7fd960096bc0 tx=0x7fd96000cdb0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 -- 172.21.15.40:0/4177394928 shutdown_connections 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7216a0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 --2- 172.21.15.40:0/4177394928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c721be0 0x55565c72c590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 -- 172.21.15.40:0/4177394928 >> 172.21.15.40:0/4177394928 conn(0x55565c717da0 msgr2=0x55565c71d6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:14.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 -- 172.21.15.40:0/4177394928 shutdown_connections 2023-07-14T11:14:14.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 -- 172.21.15.40:0/4177394928 wait complete. 2023-07-14T11:14:14.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.989+0000 7fd988418b80 1 -- start start 2023-07-14T11:14:14.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.990+0000 7fd988418b80 1 Processor -- start 2023-07-14T11:14:14.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c729dc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72a300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c613ac0 con 0x55565c72cad0 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c6137c0 con 0x55565c7212d0 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c4f7920 con 0x55565c721be0 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd988418b80 10 client.0 _ll_register_callbacks cb 0x55565c611760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96dd9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c729dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34798/0 (socket says 172.21.15.40:34798) 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 -- 172.21.15.40:0/3140684221 learned_addr learned my addr 172.21.15.40:0/3140684221 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72a300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 -- 172.21.15.40:0/3140684221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 msgr2=0x55565c729dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c729dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 -- 172.21.15.40:0/3140684221 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 msgr2=0x55565c72a300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72a300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:14.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96e59c700 1 -- 172.21.15.40:0/3140684221 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd9701a4cf0 con 0x55565c7212d0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.991+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72a300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd96e59c700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fd960099e00 tx=0x7fd96009e180 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bd39d362c9e592e8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd96009a3b0 con 0x55565c7212d0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd96009a510 con 0x55565c7212d0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55565c5cd4f0 con 0x55565c7212d0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd9600a2110 con 0x55565c7212d0 2023-07-14T11:14:14.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 10 client.26186 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x55565c48ac60 con 0x55565c7212d0 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd96007c020 con 0x55565c7212d0 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7fd960004a00 con 0x55565c7212d0 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 10 client.26186 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 10 client.26186 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 20 client.26186 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c73c220 0x55565c73e630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:14.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.992+0000 7fd988418b80 4 client.26186 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:15.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.993+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55565c66ceb0 con 0x55565c73c220 2023-07-14T11:14:15.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.993+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c73c220 0x55565c73e630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.993+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c73c220 0x55565c73e630 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=cd4476225e413084 server_cookie=3264e0b8302b6b5 in_seq=0 out_seq=0 2023-07-14T11:14:15.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.993+0000 7fd955ffb700 10 client.26186 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:15.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd955ffb700 1 -- 172.21.15.40:0/3140684221 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55565c66ceb0 con 0x55565c73c220 2023-07-14T11:14:15.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd955ffb700 10 client.26186 handle_command_reply: tid=0 2023-07-14T11:14:15.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 client.26186 shutdown 2023-07-14T11:14:15.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 msgr2=0x55565c729880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7fd960099e00 tx=0x7fd96009e180 comp rx=0 tx=0).stop 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 shutdown_connections 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c73c220 0x55565c73e630 crc :-1 s=READY pgs=75 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c72a300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c729880 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd955ffb700 0 client.26186 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 --2- 172.21.15.40:0/3140684221 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c729dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.994+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 >> 172.21.15.40:0/3140684221 conn(0x55565c717da0 msgr2=0x55565c71d230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:15.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.995+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 shutdown_connections 2023-07-14T11:14:15.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.995+0000 7fd988418b80 1 -- 172.21.15.40:0/3140684221 wait complete. 2023-07-14T11:14:15.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:14.995+0000 7fd988418b80 20 client.26186 trim_cache size 0 max 16384 2023-07-14T11:14:15.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.008+0000 7fd988418b80 1 Processor -- start 2023-07-14T11:14:15.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 -- start start 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c721be0 0x55565c72a130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 0x55565c726b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c738ec0 con 0x55565c721be0 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c73ec90 con 0x55565c72cad0 2023-07-14T11:14:15.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd988418b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c727e00 con 0x55565c7212d0 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96ed9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 0x55565c726b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96dd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c721be0 0x55565c72a130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60772/0 (socket says 172.21.15.40:60772) 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96ed9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 0x55565c726b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34808/0 (socket says 172.21.15.40:34808) 2023-07-14T11:14:15.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.009+0000 7fd96e59c700 1 -- 172.21.15.40:0/1405094426 learned_addr learned my addr 172.21.15.40:0/1405094426 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:15.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 -- 172.21.15.40:0/1405094426 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 msgr2=0x55565c726b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 0x55565c726b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 -- 172.21.15.40:0/1405094426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c721be0 msgr2=0x55565c72a130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c721be0 0x55565c72a130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 -- 172.21.15.40:0/1405094426 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55565c7195e0 con 0x55565c7212d0 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fd960096b70 tx=0x7fd9600cad50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7d66d1725ce7eb79 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd95d7fa700 1 -- 172.21.15.40:0/1405094426 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd960099850 con 0x55565c7212d0 2023-07-14T11:14:15.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd95d7fa700 1 -- 172.21.15.40:0/1405094426 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd96007c030 con 0x55565c7212d0 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.010+0000 7fd95d7fa700 1 -- 172.21.15.40:0/1405094426 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd9600a2370 con 0x55565c7212d0 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- 172.21.15.40:0/1405094426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 msgr2=0x55565c7334e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7fd960096b70 tx=0x7fd9600cad50 comp rx=0 tx=0).stop 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- 172.21.15.40:0/1405094426 shutdown_connections 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c72cad0 0x55565c726b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c7212d0 0x55565c7334e0 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 --2- 172.21.15.40:0/1405094426 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c721be0 0x55565c72a130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- 172.21.15.40:0/1405094426 >> 172.21.15.40:0/1405094426 conn(0x55565c717da0 msgr2=0x55565c724bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- 172.21.15.40:0/1405094426 shutdown_connections 2023-07-14T11:14:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- 172.21.15.40:0/1405094426 wait complete. 2023-07-14T11:14:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.011+0000 7fd988418b80 1 -- start start 2023-07-14T11:14:15.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.012+0000 7fd988418b80 1 Processor -- start 2023-07-14T11:14:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.012+0000 7fd988418b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c731260 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c7317a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c78f1d0 con 0x55565c72cad0 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c78f340 con 0x55565c7212d0 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55565c78f500 con 0x55565c721be0 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd988418b80 10 client.0 _ll_register_callbacks cb 0x7fd9600091e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34812/0 (socket says 172.21.15.40:34812) 2023-07-14T11:14:15.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 -- 172.21.15.40:0/1967357128 learned_addr learned my addr 172.21.15.40:0/1967357128 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c731260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c7317a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 -- 172.21.15.40:0/1967357128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 msgr2=0x55565c731260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c731260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 -- 172.21.15.40:0/1967357128 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 msgr2=0x55565c7317a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c7317a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96e59c700 1 -- 172.21.15.40:0/1967357128 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55565c48ac60 con 0x55565c7212d0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.013+0000 7fd96ed9d700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c7317a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd96e59c700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fd960021000 tx=0x7fd960009550 comp rx=0 tx=0).ready entity=mon.2 client_cookie=96e47a5e08abdf7e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd955ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd960004c10 con 0x55565c7212d0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd96007d080 con 0x55565c7212d0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fd9600a25a0 con 0x55565c7212d0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd9600a2110 con 0x55565c7212d0 2023-07-14T11:14:15.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd988418b80 10 client.26192 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x55565c4c8250 con 0x55565c7212d0 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.014+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd9600a25a0 con 0x55565c7212d0 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7fd9600b8330 con 0x55565c7212d0 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 10 client.26192 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 10 client.26192 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 20 client.26192 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c790730 0x55565c792b10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:15.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 4 client.26192 mds_command: new command op to 24569 tid=0[{"prefix": "damage ls"}] 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "damage ls"}) v1 -- 0x55565c62f050 con 0x55565c790730 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c790730 0x55565c792b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd96dd9b700 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c790730 0x55565c792b10 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8c6b0d245eadd5a1 server_cookie=b4dd77b41c487cbe in_seq=0 out_seq=0 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.015+0000 7fd955ffb700 10 client.26192 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd955ffb700 1 -- 172.21.15.40:0/1967357128 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+3 (crc 0 0 0) 0x55565c62f050 con 0x55565c790730 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd955ffb700 10 client.26192 handle_command_reply: tid=0 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 client.26192 shutdown 2023-07-14T11:14:15.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 msgr2=0x55565c730d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7fd960021000 tx=0x7fd960009550 comp rx=0 tx=0).stop 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 shutdown_connections 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55565c790730 0x55565c792b10 crc :-1 s=READY pgs=76 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55565c72cad0 0x55565c7317a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55565c7212d0 0x55565c730d20 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 --2- 172.21.15.40:0/1967357128 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55565c721be0 0x55565c731260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 >> 172.21.15.40:0/1967357128 conn(0x55565c717da0 msgr2=0x55565c72a060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd955ffb700 0 client.26192 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 shutdown_connections 2023-07-14T11:14:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 1 -- 172.21.15.40:0/1967357128 wait complete. 2023-07-14T11:14:15.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.016+0000 7fd988418b80 20 client.26192 trim_cache size 0 max 16384 2023-07-14T11:14:15.025 INFO:teuthology.orchestra.run.smithi040.stdout:[] 2023-07-14T11:14:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 msgr2=0x7fd9580a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 0x7fd9580a37f0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd9701000e0 tx=0x7fd964009260 comp rx=0 tx=0).stop 2023-07-14T11:14:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 msgr2=0x7fd9700fe8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7fd96000e9c0 tx=0x7fd96000ecd0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 shutdown_connections 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd9580a1340 0x7fd9580a37f0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd97010a070 0x7fd9701a3ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd970100ca0 0x7fd9700fe8f0 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 --2- 172.21.15.40:0/317243574 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd9701015b0 0x7fd9700fee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 >> 172.21.15.40:0/317243574 conn(0x7fd9700fa4b0 msgr2=0x7fd970101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 shutdown_connections 2023-07-14T11:14:15.027 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:15.017+0000 7fd9749ff700 1 -- 172.21.15.40:0/317243574 wait complete. 2023-07-14T11:14:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:15 smithi130 ceph-mon[129405]: pgmap v66: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 2.7 MiB/s wr, 367 op/s 2023-07-14T11:14:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:15 smithi040 ceph-mon[97954]: pgmap v66: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 2.7 MiB/s wr, 367 op/s 2023-07-14T11:14:15.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:15 smithi045 ceph-mon[128791]: pgmap v66: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 65 KiB/s rd, 2.7 MiB/s wr, 367 op/s 2023-07-14T11:14:16.038 INFO:tasks.fwd_scrub.fs.[cephfs]:scrubbing fs: cephfs 2023-07-14T11:14:16.038 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub start / force,recursive 2023-07-14T11:14:16.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/572444757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c981015b0 msgr2=0x7f9c98109aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/572444757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c981015b0 0x7f9c98109aa0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f9c8800dc80 tx=0x7f9c8800df90 comp rx=0 tx=0).stop 2023-07-14T11:14:16.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/572444757 shutdown_connections 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/572444757 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c9810a070 0x7f9c9810c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/572444757 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c98100ca0 0x7f9c98101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/572444757 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c981015b0 0x7f9c98109aa0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/572444757 >> 172.21.15.40:0/572444757 conn(0x7f9c980fa4b0 msgr2=0x7f9c980fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.401+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/572444757 shutdown_connections 2023-07-14T11:14:16.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.402+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/572444757 wait complete. 2023-07-14T11:14:16.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.403+0000 7f9c9f5ff700 1 Processor -- start 2023-07-14T11:14:16.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.403+0000 7f9c9f5ff700 1 -- start start 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 0x7f9c98199810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c981015b0 0x7f9c98199d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 0x7f9c98193880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c98193ef0 con 0x7f9c981015b0 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c98194060 con 0x7f9c98100ca0 2023-07-14T11:14:16.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9f5ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9c98194220 con 0x7f9c9810a070 2023-07-14T11:14:16.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9d39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 0x7f9c98199810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9d39b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 0x7f9c98199810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34838/0 (socket says 172.21.15.40:34838) 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.404+0000 7f9c9d39b700 1 -- 172.21.15.40:0/3260332373 learned_addr learned my addr 172.21.15.40:0/3260332373 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c981015b0 0x7f9c98199d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 0x7f9c98193880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 -- 172.21.15.40:0/3260332373 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 msgr2=0x7f9c98199810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 0x7f9c98199810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 -- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c981015b0 msgr2=0x7f9c98199d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c981015b0 0x7f9c98199d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 -- 172.21.15.40:0/3260332373 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9c940095f0 con 0x7f9c9810a070 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.405+0000 7f9c9db9c700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 0x7f9c98193880 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f9c9000efc0 tx=0x7f9c9000c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.406+0000 7f9c8e7fc700 1 -- 172.21.15.40:0/3260332373 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c90018070 con 0x7f9c9810a070 2023-07-14T11:14:16.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.406+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9c8800d930 con 0x7f9c9810a070 2023-07-14T11:14:16.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.406+0000 7f9c8e7fc700 1 -- 172.21.15.40:0/3260332373 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9c9000f040 con 0x7f9c9810a070 2023-07-14T11:14:16.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.406+0000 7f9c8e7fc700 1 -- 172.21.15.40:0/3260332373 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c9000cd40 con 0x7f9c9810a070 2023-07-14T11:14:16.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.406+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c981a0620 con 0x7f9c9810a070 2023-07-14T11:14:16.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.411+0000 7f9c8e7fc700 1 -- 172.21.15.40:0/3260332373 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f9c90003a00 con 0x7f9c9810a070 2023-07-14T11:14:16.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.413+0000 7f9cb2f9cb80 1 Processor -- start 2023-07-14T11:14:16.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 -- start start 2023-07-14T11:14:16.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9c8e7fc700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 0x7f9c840a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9c8e7fc700 1 -- 172.21.15.40:0/3260332373 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9c900ccac0 con 0x7f9c9810a070 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 0x5589f7373720 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f737eb50 0x5589f7380f30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7279d30 con 0x5589f7373350 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f72ccac0 con 0x5589f737eb50 2023-07-14T11:14:16.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.414+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f72c5f50 con 0x5589f7373c60 2023-07-14T11:14:16.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.415+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.415+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 0x7f9c840a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.415+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60810/0 (socket says 172.21.15.40:60810) 2023-07-14T11:14:16.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.415+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2932969216 learned_addr learned my addr 172.21.15.40:0/2932969216 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9db9c700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f737eb50 0x5589f7380f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 0x5589f7373720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2932969216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f737eb50 msgr2=0x5589f7380f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f737eb50 0x5589f7380f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2932969216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 msgr2=0x5589f7373720 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 0x5589f7373720 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2932969216 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5589f70f3e60 con 0x5589f7373c60 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 0x5589f7373720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:16.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 0x7f9c840a37f0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f9c88019000 tx=0x7f9c8800d830 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f9c8800ce70 tx=0x7f9c8802a000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=17a12a854d58f4fb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.416+0000 7f9c837fe700 1 -- 172.21.15.40:0/2932969216 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c8802a070 con 0x5589f7373c60 2023-07-14T11:14:16.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.417+0000 7f9c837fe700 1 -- 172.21.15.40:0/2932969216 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9c8800c9c0 con 0x5589f7373c60 2023-07-14T11:14:16.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.417+0000 7f9c837fe700 1 -- 172.21.15.40:0/2932969216 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c88012040 con 0x5589f7373c60 2023-07-14T11:14:16.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2932969216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 msgr2=0x5589f737e610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7f9c8800ce70 tx=0x7f9c8802a000 comp rx=0 tx=0).stop 2023-07-14T11:14:16.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2932969216 shutdown_connections 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f737eb50 0x5589f7380f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f7373350 0x5589f7373720 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2932969216 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f737e610 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2932969216 >> 172.21.15.40:0/2932969216 conn(0x5589f7369e20 msgr2=0x5589f736f720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2932969216 shutdown_connections 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.418+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2932969216 wait complete. 2023-07-14T11:14:16.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.419+0000 7f9cb2f9cb80 1 -- start start 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.419+0000 7f9cb2f9cb80 1 Processor -- start 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f737e5a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f737bf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7265b40 con 0x5589f737eb50 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7265840 con 0x5589f7373c60 2023-07-14T11:14:16.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f71499a0 con 0x5589f7373350 2023-07-14T11:14:16.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.420+0000 7f9cb2f9cb80 10 client.0 _ll_register_callbacks cb 0x5589f72637e0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:16.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34860/0 (socket says 172.21.15.40:34860) 2023-07-14T11:14:16.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2383093049 learned_addr learned my addr 172.21.15.40:0/2383093049 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2383093049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 msgr2=0x5589f737e5a0 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:14:16.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f737e5a0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2383093049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 msgr2=0x5589f737bf60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f737bf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/2383093049 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9c9810d170 con 0x5589f7373c60 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f9c88000c00 tx=0x7f9c8800f910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=7a4f77b10539f1de server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.421+0000 7f9c80ff9700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:16.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c88012070 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5589f72bef30 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9c88005610 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9c880057d0 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9cb2f9cb80 10 client.26201 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.422+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x5589f70dcce0 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5589f72bef30 con 0x5589f7373c60 2023-07-14T11:14:16.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f9c880259f0 con 0x5589f7373c60 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 10 client.26201 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 10 client.26201 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 20 client.26201 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f738e110 0x5589f7390520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 4 client.26201 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5589f72810d0 con 0x5589f738e110 2023-07-14T11:14:16.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.423+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f738e110 0x5589f7390520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.424+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f738e110 0x5589f7390520 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=279c1e9ad6ca9416 server_cookie=b95bbd9b068deb20 in_seq=0 out_seq=0 2023-07-14T11:14:16.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.424+0000 7f9c80ff9700 10 client.26201 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.425+0000 7f9c80ff9700 1 -- 172.21.15.40:0/2383093049 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5589f72810d0 con 0x5589f738e110 2023-07-14T11:14:16.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.425+0000 7f9c80ff9700 10 client.26201 handle_command_reply: tid=0 2023-07-14T11:14:16.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.425+0000 7f9cb2f9cb80 1 client.26201 shutdown 2023-07-14T11:14:16.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 msgr2=0x5589f737b820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f9c88000c00 tx=0x7f9c8800f910 comp rx=0 tx=0).stop 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 shutdown_connections 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f738e110 0x5589f7390520 crc :-1 s=READY pgs=77 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f737bf60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f737e5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/2383093049 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f737b820 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 >> 172.21.15.40:0/2383093049 conn(0x5589f7369e20 msgr2=0x5589f736f2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9c80ff9700 0 client.26201 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 shutdown_connections 2023-07-14T11:14:16.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/2383093049 wait complete. 2023-07-14T11:14:16.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.426+0000 7f9cb2f9cb80 20 client.26201 trim_cache size 0 max 16384 2023-07-14T11:14:16.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.462+0000 7f9cb2f9cb80 1 Processor -- start 2023-07-14T11:14:16.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.462+0000 7f9cb2f9cb80 1 -- start start 2023-07-14T11:14:16.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f736f580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f736e5d0 con 0x5589f737eb50 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7390bd0 con 0x5589f7373c60 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7385a70 con 0x5589f7373350 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9c9d39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34862/0 (socket says 172.21.15.40:34862) 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1667111483 learned_addr learned my addr 172.21.15.40:0/1667111483 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.463+0000 7f9c9d39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60826/0 (socket says 172.21.15.40:60826) 2023-07-14T11:14:16.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1667111483 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 msgr2=0x5589f7385530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1667111483 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 msgr2=0x5589f736f580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f736f580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1667111483 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5589f736b660 con 0x5589f7373c60 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.464+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.465+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f9c88025d50 tx=0x7f9c8800ca70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=59a7d3bbcbddcae8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.465+0000 7f9c83fff700 1 -- 172.21.15.40:0/1667111483 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c88012020 con 0x5589f7373c60 2023-07-14T11:14:16.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.465+0000 7f9c83fff700 1 -- 172.21.15.40:0/1667111483 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f9c88022420 con 0x5589f7373c60 2023-07-14T11:14:16.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.465+0000 7f9c83fff700 1 -- 172.21.15.40:0/1667111483 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c8802a040 con 0x5589f7373c60 2023-07-14T11:14:16.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1667111483 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 msgr2=0x5589f736f040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f9c88025d50 tx=0x7f9c8800ca70 comp rx=0 tx=0).stop 2023-07-14T11:14:16.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1667111483 shutdown_connections 2023-07-14T11:14:16.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f736f580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373350 0x5589f7385530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1667111483 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373c60 0x5589f736f040 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1667111483 >> 172.21.15.40:0/1667111483 conn(0x5589f7369e20 msgr2=0x5589f736e930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1667111483 shutdown_connections 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.466+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1667111483 wait complete. 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.467+0000 7f9cb2f9cb80 1 -- start start 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.467+0000 7f9cb2f9cb80 1 Processor -- start 2023-07-14T11:14:16.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 0x5589f7383cc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f73e1fe0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f73849b0 con 0x5589f737eb50 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f7384b20 con 0x5589f7373350 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5589f73e2550 con 0x5589f7373c60 2023-07-14T11:14:16.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.468+0000 7f9cb2f9cb80 10 client.0 _ll_register_callbacks cb 0x7f9c8800cf80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:16.473 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60836/0 (socket says 172.21.15.40:60836) 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1436194440 learned_addr learned my addr 172.21.15.40:0/1436194440 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 0x5589f7383cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 msgr2=0x5589f7383cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 0x5589f7383cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1436194440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 msgr2=0x5589f73e1fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f73e1fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.469+0000 7f9c9cb9a700 1 -- 172.21.15.40:0/1436194440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5589f70dcce0 con 0x5589f7373c60 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 0x5589f7383cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9c9cb9a700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f9c8800d6b0 tx=0x7f9c88022590 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c782bbd711ecae49 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9c80ff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f9c88012070 con 0x5589f7373c60 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f9c88003df0 con 0x5589f7373c60 2023-07-14T11:14:16.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.470+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5589f721f570 con 0x5589f7373c60 2023-07-14T11:14:16.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.471+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f9c8802a070 con 0x5589f7373c60 2023-07-14T11:14:16.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.471+0000 7f9cb2f9cb80 10 client.26170 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:16.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.471+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x5589f711a2d0 con 0x5589f7373c60 2023-07-14T11:14:16.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.471+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5589f721f570 con 0x5589f7373c60 2023-07-14T11:14:16.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f9c88025460 con 0x5589f7373c60 2023-07-14T11:14:16.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 10 client.26170 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:16.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 10 client.26170 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:16.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 20 client.26170 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:16.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f73e3860 0x5589f73e5c40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 4 client.26170 mds_command: new command op to 24569 tid=0[{"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}] 2023-07-14T11:14:16.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.472+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub start", "path": "/", "scrubops": ["force", "recursive"]}) v1 -- 0x5589f7114ff0 con 0x5589f73e3860 2023-07-14T11:14:16.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.473+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f73e3860 0x5589f73e5c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.473+0000 7f9c9d39b700 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f73e3860 0x5589f73e5c40 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2f71abfce1dfe8c server_cookie=7ae25972593538e4 in_seq=0 out_seq=0 2023-07-14T11:14:16.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.473+0000 7f9c80ff9700 10 client.26170 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.476+0000 7f9c80ff9700 1 -- 172.21.15.40:0/1436194440 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+110 (crc 0 0 0) 0x5589f7114ff0 con 0x5589f73e3860 2023-07-14T11:14:16.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.476+0000 7f9c80ff9700 10 client.26170 handle_command_reply: tid=0 2023-07-14T11:14:16.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.476+0000 7f9cb2f9cb80 1 client.26170 shutdown 2023-07-14T11:14:16.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.476+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 msgr2=0x5589f7384200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.476+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7f9c8800d6b0 tx=0x7f9c88022590 comp rx=0 tx=0).stop 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 shutdown_connections 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5589f73e3860 0x5589f73e5c40 crc :-1 s=READY pgs=78 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5589f737eb50 0x5589f73e1fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5589f7373350 0x5589f7383cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 --2- 172.21.15.40:0/1436194440 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5589f7373c60 0x5589f7384200 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 >> 172.21.15.40:0/1436194440 conn(0x5589f7369e20 msgr2=0x5589f736f1c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9c80ff9700 0 client.26170 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 shutdown_connections 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 1 -- 172.21.15.40:0/1436194440 wait complete. 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.477+0000 7f9cb2f9cb80 20 client.26170 trim_cache size 0 max 16384 2023-07-14T11:14:16.481 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:14:16.482 INFO:teuthology.orchestra.run.smithi040.stdout: "return_code": 0, 2023-07-14T11:14:16.482 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:14:16.482 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": "asynchronous" 2023-07-14T11:14:16.482 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 msgr2=0x7f9c840a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 0x7f9c840a37f0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7f9c88019000 tx=0x7f9c8800d830 comp rx=0 tx=0).stop 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 msgr2=0x7f9c98193880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 0x7f9c98193880 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f9c9000efc0 tx=0x7f9c9000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 shutdown_connections 2023-07-14T11:14:16.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f9c840a1340 0x7f9c840a37f0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f9c9810a070 0x7f9c98193880 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f9c98100ca0 0x7f9c98199810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 --2- 172.21.15.40:0/3260332373 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9c981015b0 0x7f9c98199d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.478+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 >> 172.21.15.40:0/3260332373 conn(0x7f9c980fa4b0 msgr2=0x7f9c98101a40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.479+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 shutdown_connections 2023-07-14T11:14:16.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.479+0000 7f9c9f5ff700 1 -- 172.21.15.40:0/3260332373 wait complete. 2023-07-14T11:14:16.512 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:14:16.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 -- 172.21.15.40:0/3380838545 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 msgr2=0x7f13f8109aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3380838545 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8109aa0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f13ec009990 tx=0x7f13ec009ca0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 -- 172.21.15.40:0/3380838545 shutdown_connections 2023-07-14T11:14:16.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3380838545 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f810c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3380838545 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 0x7f13f8101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3380838545 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8109aa0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 -- 172.21.15.40:0/3380838545 >> 172.21.15.40:0/3380838545 conn(0x7f13f80fa4b0 msgr2=0x7f13f80fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 -- 172.21.15.40:0/3380838545 shutdown_connections 2023-07-14T11:14:16.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.766+0000 7f13fdfff700 1 -- 172.21.15.40:0/3380838545 wait complete. 2023-07-14T11:14:16.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.768+0000 7f13fdfff700 1 Processor -- start 2023-07-14T11:14:16.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.768+0000 7f13fdfff700 1 -- start start 2023-07-14T11:14:16.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 0x7f13f8197e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.770 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8198370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f819c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13f819ce10 con 0x7f13f8100ca0 2023-07-14T11:14:16.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13f819cf80 con 0x7f13f810a070 2023-07-14T11:14:16.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13fdfff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13f819d140 con 0x7f13f81015b0 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13f6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8198370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13f6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8198370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60854/0 (socket says 172.21.15.40:60854) 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13f6ffd700 1 -- 172.21.15.40:0/3806315553 learned_addr learned my addr 172.21.15.40:0/3806315553 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 0x7f13f8197e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.769+0000 7f13f7fff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f819c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 -- 172.21.15.40:0/3806315553 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 msgr2=0x7f13f8198370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8198370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 -- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 msgr2=0x7f13f8197e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 0x7f13f8197e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 -- 172.21.15.40:0/3806315553 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13ec0095f0 con 0x7f13f810a070 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f7fff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f819c7d0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f13f000efc0 tx=0x7f13f000c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13f4ff9700 1 -- 172.21.15.40:0/3806315553 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13f0018070 con 0x7f13f810a070 2023-07-14T11:14:16.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.770+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13f819d3c0 con 0x7f13f810a070 2023-07-14T11:14:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.771+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13f81a4c60 con 0x7f13f810a070 2023-07-14T11:14:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.771+0000 7f13f4ff9700 1 -- 172.21.15.40:0/3806315553 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f13f000f040 con 0x7f13f810a070 2023-07-14T11:14:16.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.771+0000 7f13f4ff9700 1 -- 172.21.15.40:0/3806315553 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13f000cd40 con 0x7f13f810a070 2023-07-14T11:14:16.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.775+0000 7f13f4ff9700 1 -- 172.21.15.40:0/3806315553 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f13f0003a00 con 0x7f13f810a070 2023-07-14T11:14:16.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.776+0000 7f13f4ff9700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 0x7f13e00a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.776+0000 7f13f4ff9700 1 -- 172.21.15.40:0/3806315553 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f13f00cca70 con 0x7f13f810a070 2023-07-14T11:14:16.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.777+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 0x7f13e00a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.777+0000 7f141184cb80 1 Processor -- start 2023-07-14T11:14:16.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 0x7f13e00a37f0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f13e8009c60 tx=0x7f13e8009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 -- start start 2023-07-14T11:14:16.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e1aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7d13cb0 con 0x5566c7e18ad0 2023-07-14T11:14:16.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7d66a40 con 0x5566c7e0d2d0 2023-07-14T11:14:16.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f141184cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7d5fed0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.778+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34914/0 (socket says 172.21.15.40:34914) 2023-07-14T11:14:16.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 -- 172.21.15.40:0/3112005364 learned_addr learned my addr 172.21.15.40:0/3112005364 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f7fff700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e1aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 -- 172.21.15.40:0/3112005364 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 msgr2=0x5566c7e18590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 -- 172.21.15.40:0/3112005364 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 msgr2=0x5566c7e1aeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e1aeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 -- 172.21.15.40:0/3112005364 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5566c7b8dde0 con 0x5566c7e0d2d0 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.779+0000 7f13f77fe700 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f13e800e4e0 tx=0x7f13e8014000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f19fb90613dd57b7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.780+0000 7f13ddffb700 1 -- 172.21.15.40:0/3112005364 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e800ee00 con 0x5566c7e0d2d0 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.780+0000 7f13ddffb700 1 -- 172.21.15.40:0/3112005364 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f13e8021070 con 0x5566c7e0d2d0 2023-07-14T11:14:16.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.780+0000 7f13ddffb700 1 -- 172.21.15.40:0/3112005364 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e801d810 con 0x5566c7e0d2d0 2023-07-14T11:14:16.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 -- 172.21.15.40:0/3112005364 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 msgr2=0x5566c7e0d6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f13e800e4e0 tx=0x7f13e8014000 comp rx=0 tx=0).stop 2023-07-14T11:14:16.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 -- 172.21.15.40:0/3112005364 shutdown_connections 2023-07-14T11:14:16.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e1aeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0d2d0 0x5566c7e0d6a0 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 --2- 172.21.15.40:0/3112005364 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 -- 172.21.15.40:0/3112005364 >> 172.21.15.40:0/3112005364 conn(0x5566c7e03da0 msgr2=0x5566c7e096a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 -- 172.21.15.40:0/3112005364 shutdown_connections 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.781+0000 7f141184cb80 1 -- 172.21.15.40:0/3112005364 wait complete. 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.782+0000 7f141184cb80 1 -- start start 2023-07-14T11:14:16.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.782+0000 7f141184cb80 1 Processor -- start 2023-07-14T11:14:16.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e15780 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e15ec0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7cffac0 con 0x5566c7e18ad0 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7cff7c0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7be3920 con 0x5566c7e0d2d0 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.783+0000 7f141184cb80 10 client.0 _ll_register_callbacks cb 0x5566c7cfd760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:16.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60876/0 (socket says 172.21.15.40:60876) 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 -- 172.21.15.40:0/704362182 learned_addr learned my addr 172.21.15.40:0/704362182 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e15780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f7fff700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e15ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 -- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 msgr2=0x5566c7e15780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e15780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 -- 172.21.15.40:0/704362182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 msgr2=0x5566c7e15ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e15ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f77fe700 1 -- 172.21.15.40:0/704362182 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13f810d170 con 0x5566c7e0d2d0 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.784+0000 7f13f7fff700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e15ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f13f77fe700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13e800e4b0 tx=0x7f13e800edf0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=95893cec73db06cb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f13d6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e8005410 con 0x5566c7e0d2d0 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f13e8005570 con 0x5566c7e0d2d0 2023-07-14T11:14:16.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5566c7cb94f0 con 0x5566c7e0d2d0 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.785+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f13e8003680 con 0x5566c7e0d2d0 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 10 client.26182 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x7f13e8017070 con 0x5566c7e0d2d0 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5566c7cb94f0 con 0x5566c7e0d2d0 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f13e8036070 con 0x5566c7e0d2d0 2023-07-14T11:14:16.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 10 client.26182 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 10 client.26182 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 20 client.26182 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e283a0 0x5566c7e2a7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 4 client.26182 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.786+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5566c7d58eb0 con 0x5566c7e283a0 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.787+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e283a0 0x5566c7e2a7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.787+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e283a0 0x5566c7e2a7b0 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5dae67b5537d41a8 server_cookie=ad7588ccc8fcefba in_seq=0 out_seq=0 2023-07-14T11:14:16.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.787+0000 7f13d6ffd700 10 client.26182 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.788+0000 7f13d6ffd700 1 -- 172.21.15.40:0/704362182 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5566c7d58eb0 con 0x5566c7e283a0 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.788+0000 7f13d6ffd700 10 client.26182 handle_command_reply: tid=0 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.788+0000 7f141184cb80 1 client.26182 shutdown 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 msgr2=0x5566c7e18500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13e800e4b0 tx=0x7f13e800edf0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 shutdown_connections 2023-07-14T11:14:16.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e283a0 0x5566c7e2a7b0 crc :-1 s=READY pgs=79 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18ad0 0x5566c7e15ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e0d2d0 0x5566c7e18500 secure :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f13e800e4b0 tx=0x7f13e800edf0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 --2- 172.21.15.40:0/704362182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e15780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 >> 172.21.15.40:0/704362182 conn(0x5566c7e03da0 msgr2=0x5566c7e09230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f13d6ffd700 0 client.26182 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 shutdown_connections 2023-07-14T11:14:16.794 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.789+0000 7f141184cb80 1 -- 172.21.15.40:0/704362182 wait complete. 2023-07-14T11:14:16.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.790+0000 7f141184cb80 20 client.26182 trim_cache size 0 max 16384 2023-07-14T11:14:16.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 Processor -- start 2023-07-14T11:14:16.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 -- start start 2023-07-14T11:14:16.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e10a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e18570 0x5566c7e10fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e168c0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e0e540 con 0x5566c7e18570 2023-07-14T11:14:16.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.808+0000 7f141184cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e0dfb0 con 0x5566c7e18ad0 2023-07-14T11:14:16.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e10a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:41266/0 (socket says 172.21.15.40:41266) 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 -- 172.21.15.40:0/4066544032 learned_addr learned my addr 172.21.15.40:0/4066544032 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 -- 172.21.15.40:0/4066544032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 msgr2=0x5566c7e10a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e10a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 -- 172.21.15.40:0/4066544032 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e18570 msgr2=0x5566c7e10fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e18570 0x5566c7e10fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f77fe700 1 -- 172.21.15.40:0/4066544032 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5566c7b76c60 con 0x5566c7e0dbe0 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.809+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e10a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.810+0000 7f13f77fe700 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f13e8016040 tx=0x7f13e800ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3f7a02d948bae3c8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.810+0000 7f13de7fc700 1 -- 172.21.15.40:0/4066544032 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e80092d0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.810+0000 7f13de7fc700 1 -- 172.21.15.40:0/4066544032 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f13e801cea0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.810+0000 7f13de7fc700 1 -- 172.21.15.40:0/4066544032 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e8003950 con 0x5566c7e0dbe0 2023-07-14T11:14:16.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 -- 172.21.15.40:0/4066544032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 msgr2=0x5566c7e18030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 crypto rx=0x7f13e8016040 tx=0x7f13e800ef10 comp rx=0 tx=0).stop 2023-07-14T11:14:16.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 -- 172.21.15.40:0/4066544032 shutdown_connections 2023-07-14T11:14:16.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e18570 0x5566c7e10fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e0dbe0 0x5566c7e18030 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 --2- 172.21.15.40:0/4066544032 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e10a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 -- 172.21.15.40:0/4066544032 >> 172.21.15.40:0/4066544032 conn(0x5566c7e03da0 msgr2=0x5566c7e2a3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.811+0000 7f141184cb80 1 -- 172.21.15.40:0/4066544032 shutdown_connections 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.812+0000 7f141184cb80 1 -- 172.21.15.40:0/4066544032 wait complete. 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.812+0000 7f141184cb80 1 -- start start 2023-07-14T11:14:16.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.812+0000 7f141184cb80 1 Processor -- start 2023-07-14T11:14:16.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18570 0x5566c7e1d7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e1dd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e1e2e0 con 0x5566c7e18570 2023-07-14T11:14:16.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e1e450 con 0x5566c7e0dbe0 2023-07-14T11:14:16.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5566c7e1e610 con 0x5566c7e18ad0 2023-07-14T11:14:16.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.814+0000 7f141184cb80 10 client.0 _ll_register_callbacks cb 0x7f13e8005ec0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:16.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34940/0 (socket says 172.21.15.40:34940) 2023-07-14T11:14:16.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 -- 172.21.15.40:0/2545991505 learned_addr learned my addr 172.21.15.40:0/2545991505 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:16.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18570 0x5566c7e1d7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f7fff700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e1dd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 -- 172.21.15.40:0/2545991505 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 msgr2=0x5566c7e1dd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e1dd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 -- 172.21.15.40:0/2545991505 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18570 msgr2=0x5566c7e1d7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18570 0x5566c7e1d7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f77fe700 1 -- 172.21.15.40:0/2545991505 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13ec005160 con 0x5566c7e0dbe0 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.815+0000 7f13f7fff700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e1dd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f13f77fe700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f13e801ce70 tx=0x7f13e80050c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e4b4926c83239598 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:16.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f13d6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13e80056e0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f13e8021070 con 0x5566c7e0dbe0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f13e801c410 con 0x5566c7e0dbe0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f13e801c5d0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f141184cb80 10 client.26222 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.816+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x7f13e8017070 con 0x5566c7e0dbe0 2023-07-14T11:14:16.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f13e8021070 con 0x5566c7e0dbe0 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f13e800e7f0 con 0x5566c7e0dbe0 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 10 client.26222 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 10 client.26222 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 20 client.26222 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e7d440 0x5566c7e7f850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:16.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 4 client.26222 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5566c7d1b050 con 0x5566c7e7d440 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.817+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e7d440 0x5566c7e7f850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f13f6ffd700 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e7d440 0x5566c7e7f850 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=96375581bcc3eb8 server_cookie=c0a99d997cb02f56 in_seq=0 out_seq=0 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f13d6ffd700 10 client.26222 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f13d6ffd700 1 -- 172.21.15.40:0/2545991505 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x5566c7d1b050 con 0x5566c7e7d440 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f13d6ffd700 10 client.26222 handle_command_reply: tid=0 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 client.26222 shutdown 2023-07-14T11:14:16.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 msgr2=0x5566c7e1f4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f13e801ce70 tx=0x7f13e80050c0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 shutdown_connections 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5566c7e7d440 0x5566c7e7f850 crc :-1 s=READY pgs=80 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5566c7e18ad0 0x5566c7e1dd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5566c7e0dbe0 0x5566c7e1f4a0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.818+0000 7f141184cb80 1 --2- 172.21.15.40:0/2545991505 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5566c7e18570 0x5566c7e1d7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.819+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 >> 172.21.15.40:0/2545991505 conn(0x5566c7e03da0 msgr2=0x5566c7e10a90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.819+0000 7f13d6ffd700 0 client.26222 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.819+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 shutdown_connections 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.819+0000 7f141184cb80 1 -- 172.21.15.40:0/2545991505 wait complete. 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.819+0000 7f141184cb80 20 client.26222 trim_cache size 0 max 16384 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:14:16.825 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (3 inodes in the stack)", 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:14:16.826 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:14:16.827 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:14:16.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 msgr2=0x7f13e00a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 0x7f13e00a37f0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f13e8009c60 tx=0x7f13e8009260 comp rx=0 tx=0).stop 2023-07-14T11:14:16.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 msgr2=0x7f13f819c7d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f819c7d0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f13f000efc0 tx=0x7f13f000c3a0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 shutdown_connections 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f13e00a1340 0x7f13e00a37f0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f13f810a070 0x7f13f819c7d0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13f8100ca0 0x7f13f8197e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 --2- 172.21.15.40:0/3806315553 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f13f81015b0 0x7f13f8198370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 >> 172.21.15.40:0/3806315553 conn(0x7f13f80fa4b0 msgr2=0x7f13f810c180 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:16.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 shutdown_connections 2023-07-14T11:14:16.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:16.820+0000 7f13fdfff700 1 -- 172.21.15.40:0/3806315553 wait complete. 2023-07-14T11:14:16.840 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:14:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub queued for path: / 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: pgmap v67: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:18.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:17 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub queued for path: / 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: pgmap v67: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:18.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:17 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:14:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub queued for path: / 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: pgmap v67: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:18.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:17 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:14:19.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:18 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:18 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:14:19.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:18 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:18 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:14:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:18 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:14:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:18 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:14:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:19 smithi130 ceph-mon[129405]: pgmap v68: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:19 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:19 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:19 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:19 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:19 smithi040 ceph-mon[97954]: pgmap v68: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:19 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:19 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:19 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:19 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:19 smithi045 ceph-mon[128791]: pgmap v68: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 1.7 MiB/s wr, 262 op/s 2023-07-14T11:14:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:19 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:19 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:19 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:19 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:21.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:20 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:14:21.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:20 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:14:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:20 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:21.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:20 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:14:21.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:20 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:14:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:20 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:21.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:20 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:14:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:20 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:14:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:20 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:14:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:21 smithi130 ceph-mon[129405]: pgmap v69: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 1.8 MiB/s wr, 285 op/s 2023-07-14T11:14:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:21 smithi040 ceph-mon[97954]: pgmap v69: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 1.8 MiB/s wr, 285 op/s 2023-07-14T11:14:22.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:21 smithi045 ceph-mon[128791]: pgmap v69: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 1.8 MiB/s wr, 285 op/s 2023-07-14T11:14:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:23 smithi130 ceph-mon[129405]: pgmap v70: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 961 KiB/s wr, 156 op/s 2023-07-14T11:14:24.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:23 smithi040 ceph-mon[97954]: pgmap v70: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 961 KiB/s wr, 156 op/s 2023-07-14T11:14:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:23 smithi045 ceph-mon[128791]: pgmap v70: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 961 KiB/s wr, 156 op/s 2023-07-14T11:14:25.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:25 smithi130 ceph-mon[129405]: pgmap v71: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 967 KiB/s wr, 162 op/s 2023-07-14T11:14:25.807 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:25 smithi040 ceph-mon[97954]: pgmap v71: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 967 KiB/s wr, 162 op/s 2023-07-14T11:14:25.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:25 smithi045 ceph-mon[128791]: pgmap v71: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 967 KiB/s wr, 162 op/s 2023-07-14T11:14:27.417 INFO:teuthology.orchestra.run.smithi040.stderr:35000000 of 50000000 tuples (70%) done (elapsed 121.91 s, remaining 52.25 s) 2023-07-14T11:14:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:27 smithi130 ceph-mon[129405]: pgmap v72: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:27 smithi040 ceph-mon[97954]: pgmap v72: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:27 smithi045 ceph-mon[128791]: pgmap v72: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:29 smithi130 ceph-mon[129405]: pgmap v73: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:30.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:29 smithi040 ceph-mon[97954]: pgmap v73: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:29 smithi045 ceph-mon[128791]: pgmap v73: 129 pgs: 129 active+clean; 6.3 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 73 KiB/s wr, 29 op/s 2023-07-14T11:14:31.954 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:31 smithi040 ceph-mon[97954]: pgmap v74: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 40 op/s 2023-07-14T11:14:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:31 smithi130 ceph-mon[129405]: pgmap v74: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 40 op/s 2023-07-14T11:14:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:31 smithi045 ceph-mon[128791]: pgmap v74: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 15 MiB/s wr, 40 op/s 2023-07-14T11:14:32.846 INFO:teuthology.orchestra.run.smithi040.stderr:35100000 of 50000000 tuples (70%) done (elapsed 127.34 s, remaining 54.06 s) 2023-07-14T11:14:33.542 INFO:teuthology.orchestra.run.smithi040.stderr:35200000 of 50000000 tuples (70%) done (elapsed 128.03 s, remaining 53.83 s) 2023-07-14T11:14:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:33 smithi130 ceph-mon[129405]: pgmap v75: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 15 MiB/s wr, 18 op/s 2023-07-14T11:14:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:33 smithi040 ceph-mon[97954]: pgmap v75: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 15 MiB/s wr, 18 op/s 2023-07-14T11:14:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:33 smithi045 ceph-mon[128791]: pgmap v75: 129 pgs: 129 active+clean; 6.4 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 24 KiB/s rd, 15 MiB/s wr, 18 op/s 2023-07-14T11:14:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:35 smithi130 ceph-mon[129405]: pgmap v76: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 41 op/s 2023-07-14T11:14:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:35 smithi040 ceph-mon[97954]: pgmap v76: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 41 op/s 2023-07-14T11:14:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:35 smithi045 ceph-mon[128791]: pgmap v76: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 31 KiB/s rd, 17 MiB/s wr, 41 op/s 2023-07-14T11:14:36.906 INFO:teuthology.orchestra.run.smithi040.stderr:PQputline failed 2023-07-14T11:14:36.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:14:36.917 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/task/exec.py", line 66, in task remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi040 with status 1: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- pgbench -s 500 -i'" 2023-07-14T11:14:37.818 ERROR:teuthology.run_tasks: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=98a6da5322b64d0d8a354d268a14b0f7 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/run_tasks.py", line 84, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/task/exec.py", line 66, in task remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi040 with status 1: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo -u postgres -- pgbench -s 500 -i'" 2023-07-14T11:14:37.831 DEBUG:teuthology.run_tasks:Unwinding manager full_sequential_finally 2023-07-14T11:14:37.842 INFO:teuthology.task.full_sequential_finally:In full_sequential_finally, running task exec... 2023-07-14T11:14:37.842 INFO:teuthology.task.exec:Executing custom commands... 2023-07-14T11:14:37.843 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:14:37.843 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'date +%s > END_TIME' 2023-07-14T11:14:37.952 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:37 smithi040 ceph-mon[97954]: pgmap v77: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:37.972 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'START_TIME=$(cat START_TIME); END_TIME=$(cat END_TIME); DIFF_TIME=$((600-(END_TIME-START_TIME))); if [ "$DIFF_TIME" -gt 0 ]; then sleep "$DIFF_TIME"; fi' 2023-07-14T11:14:38.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:37 smithi130 ceph-mon[129405]: pgmap v77: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:38.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:37 smithi045 ceph-mon[128791]: pgmap v77: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:39 smithi130 ceph-mon[129405]: pgmap v78: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:39 smithi040 ceph-mon[97954]: pgmap v78: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:39 smithi045 ceph-mon[128791]: pgmap v78: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 KiB/s rd, 17 MiB/s wr, 34 op/s 2023-07-14T11:14:41.972 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:41 smithi040 ceph-mon[97954]: pgmap v79: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 24 MiB/s wr, 43 op/s 2023-07-14T11:14:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:41 smithi130 ceph-mon[129405]: pgmap v79: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 24 MiB/s wr, 43 op/s 2023-07-14T11:14:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:41 smithi045 ceph-mon[128791]: pgmap v79: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 24 MiB/s wr, 43 op/s 2023-07-14T11:14:44.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:43 smithi130 ceph-mon[129405]: pgmap v80: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 8.7 MiB/s wr, 31 op/s 2023-07-14T11:14:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:43 smithi040 ceph-mon[97954]: pgmap v80: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 8.7 MiB/s wr, 31 op/s 2023-07-14T11:14:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:43 smithi045 ceph-mon[128791]: pgmap v80: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 8.7 MiB/s wr, 31 op/s 2023-07-14T11:14:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:45 smithi130 ceph-mon[129405]: pgmap v81: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 8.7 MiB/s wr, 39 op/s 2023-07-14T11:14:45.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:45 smithi040 ceph-mon[97954]: pgmap v81: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 8.7 MiB/s wr, 39 op/s 2023-07-14T11:14:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:45 smithi045 ceph-mon[128791]: pgmap v81: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 8.7 MiB/s wr, 39 op/s 2023-07-14T11:14:46.841 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:14:47.055 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 -- 172.21.15.40:0/1479988993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 msgr2=0x7f3f4810c920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1479988993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f4810c920 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7f3f4000b0a0 tx=0x7f3f4000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 -- 172.21.15.40:0/1479988993 shutdown_connections 2023-07-14T11:14:47.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1479988993 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f4810c920 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.056 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1479988993 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f48105af0 0x7f3f48105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1479988993 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f480ffae0 0x7f3f480fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 -- 172.21.15.40:0/1479988993 >> 172.21.15.40:0/1479988993 conn(0x7f3f480fb5e0 msgr2=0x7f3f480fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.052+0000 7f3f4efff700 1 -- 172.21.15.40:0/1479988993 shutdown_connections 2023-07-14T11:14:47.057 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.053+0000 7f3f4efff700 1 -- 172.21.15.40:0/1479988993 wait complete. 2023-07-14T11:14:47.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.054+0000 7f3f4efff700 1 Processor -- start 2023-07-14T11:14:47.059 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.054+0000 7f3f4efff700 1 -- start start 2023-07-14T11:14:47.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f48196270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48105af0 0x7f3f4819a6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.060 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819ad10 con 0x7f3f48105af0 2023-07-14T11:14:47.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819ae80 con 0x7f3f480ffae0 2023-07-14T11:14:47.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4efff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f4819b040 con 0x7f3f48100490 2023-07-14T11:14:47.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.061 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50590/0 (socket says 172.21.15.40:50590) 2023-07-14T11:14:47.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/1435079613 learned_addr learned my addr 172.21.15.40:0/1435079613 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:47.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f47fff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f48196270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48105af0 0x7f3f4819a6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/1435079613 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 msgr2=0x7f3f48196270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.062 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f48196270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48105af0 msgr2=0x7f3f4819a6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.055+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48105af0 0x7f3f4819a6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/1435079613 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f40009d00 con 0x7f3f480ffae0 2023-07-14T11:14:47.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c00e9a0 tx=0x7f3f3c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.063 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f45ffb700 1 -- 172.21.15.40:0/1435079613 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c009790 con 0x7f3f480ffae0 2023-07-14T11:14:47.064 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f45ffb700 1 -- 172.21.15.40:0/1435079613 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f3c00cbd0 con 0x7f3f480ffae0 2023-07-14T11:14:47.064 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f4819b320 con 0x7f3f480ffae0 2023-07-14T11:14:47.064 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f45ffb700 1 -- 172.21.15.40:0/1435079613 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c018680 con 0x7f3f480ffae0 2023-07-14T11:14:47.065 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.056+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f481a2b60 con 0x7f3f480ffae0 2023-07-14T11:14:47.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.060+0000 7f3f45ffb700 1 -- 172.21.15.40:0/1435079613 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3f3c00cd40 con 0x7f3f480ffae0 2023-07-14T11:14:47.068 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.061+0000 7f3f6283cb80 1 Processor -- start 2023-07-14T11:14:47.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.061+0000 7f3f6283cb80 1 -- start start 2023-07-14T11:14:47.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcab6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcbd2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.069 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afbb1cb0 con 0x5570afcab2d0 2023-07-14T11:14:47.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afc04a40 con 0x5570afcabbe0 2023-07-14T11:14:47.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afbfded0 con 0x5570afcbaee0 2023-07-14T11:14:47.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.070 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50592/0 (socket says 172.21.15.40:50592) 2023-07-14T11:14:47.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2487703573 learned_addr learned my addr 172.21.15.40:0/2487703573 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:47.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcab6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.062+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcbd2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2487703573 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 msgr2=0x5570afcbd2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.071 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcbd2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2487703573 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 msgr2=0x5570afcab6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcab6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f45ffb700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 0x7f3f380a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2487703573 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5570afa2bde0 con 0x5570afcabbe0 2023-07-14T11:14:47.072 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcbd2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:47.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f45ffb700 1 -- 172.21.15.40:0/1435079613 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3f3c014070 con 0x7f3f480ffae0 2023-07-14T11:14:47.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c0ca5a0 tx=0x7f3f3c096320 comp rx=0 tx=0).ready entity=mon.2 client_cookie=37cfbfff714dfae7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 0x7f3f380a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f32ffd700 1 -- 172.21.15.40:0/2487703573 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c099db0 con 0x5570afcabbe0 2023-07-14T11:14:47.073 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f32ffd700 1 -- 172.21.15.40:0/2487703573 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f3c099b20 con 0x5570afcabbe0 2023-07-14T11:14:47.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.063+0000 7f3f32ffd700 1 -- 172.21.15.40:0/2487703573 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c0a69e0 con 0x5570afcabbe0 2023-07-14T11:14:47.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 0x7f3f380a3770 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3f40009850 tx=0x7f3f40009780 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2487703573 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 msgr2=0x5570afcba9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c0ca5a0 tx=0x7f3f3c096320 comp rx=0 tx=0).stop 2023-07-14T11:14:47.074 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2487703573 shutdown_connections 2023-07-14T11:14:47.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcbd2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcab6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2487703573 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcba9a0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.064+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2487703573 >> 172.21.15.40:0/2487703573 conn(0x5570afca1da0 msgr2=0x5570afca76a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.075 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.065+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2487703573 shutdown_connections 2023-07-14T11:14:47.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.065+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2487703573 wait complete. 2023-07-14T11:14:47.076 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.065+0000 7f3f6283cb80 1 -- start start 2023-07-14T11:14:47.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 Processor -- start 2023-07-14T11:14:47.079 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb7be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afb9dac0 con 0x5570afcabbe0 2023-07-14T11:14:47.080 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.066+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afb9d7c0 con 0x5570afcab2d0 2023-07-14T11:14:47.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afa81920 con 0x5570afcbaee0 2023-07-14T11:14:47.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f6283cb80 10 client.0 _ll_register_callbacks cb 0x5570afb9b760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:47.081 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f47fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37670/0 (socket says 172.21.15.40:37670) 2023-07-14T11:14:47.082 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50596/0 (socket says 172.21.15.40:50596) 2023-07-14T11:14:47.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 -- 172.21.15.40:0/480877109 learned_addr learned my addr 172.21.15.40:0/480877109 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:47.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 -- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 msgr2=0x5570afcba960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 -- 172.21.15.40:0/480877109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 msgr2=0x5570afcb7be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.083 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb7be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4d59c700 1 -- 172.21.15.40:0/480877109 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f340098e0 con 0x5570afcbaee0 2023-07-14T11:14:47.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.067+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:14:47.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f3f40015000 tx=0x7f3f400099e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=48445910dd5680fe server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.084 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f27fff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:14:47.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f4000e040 con 0x5570afcbaee0 2023-07-14T11:14:47.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3f40022070 con 0x5570afcbaee0 2023-07-14T11:14:47.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5570afb574f0 con 0x5570afcbaee0 2023-07-14T11:14:47.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3f400328d0 con 0x5570afcbaee0 2023-07-14T11:14:47.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f6283cb80 10 client.26197 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:47.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x7f3f481a2b60 con 0x5570afcbaee0 2023-07-14T11:14:47.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.068+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5570afb574f0 con 0x5570afcbaee0 2023-07-14T11:14:47.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f3f40005040 con 0x5570afcbaee0 2023-07-14T11:14:47.086 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 10 client.26197 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:47.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 10 client.26197 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:47.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 20 client.26197 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:47.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afcca890 0x5570afcccca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 4 client.26197 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:14:47.087 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5570afbf6eb0 con 0x5570afcca890 2023-07-14T11:14:47.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.069+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afcca890 0x5570afcccca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.070+0000 7f3f27fff700 10 client.26197 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:47.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.070+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afcca890 0x5570afcccca0 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=5e09dc833ab26f92 server_cookie=3326fcf3bcb83c4 in_seq=0 out_seq=0 2023-07-14T11:14:47.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f27fff700 1 -- 172.21.15.40:0/480877109 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5570afbf6eb0 con 0x5570afcca890 2023-07-14T11:14:47.088 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f27fff700 10 client.26197 handle_command_reply: tid=0 2023-07-14T11:14:47.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 client.26197 shutdown 2023-07-14T11:14:47.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 msgr2=0x5570afcb82d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7f3f40015000 tx=0x7f3f400099e0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 shutdown_connections 2023-07-14T11:14:47.089 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afcca890 0x5570afcccca0 crc :-1 s=READY pgs=81 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afcb82d0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcab2d0 0x5570afcba960 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f27fff700 0 client.26197 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:47.090 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/480877109 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb7be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 >> 172.21.15.40:0/480877109 conn(0x5570afca1da0 msgr2=0x5570afca7230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.071+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 shutdown_connections 2023-07-14T11:14:47.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.072+0000 7f3f6283cb80 1 -- 172.21.15.40:0/480877109 wait complete. 2023-07-14T11:14:47.091 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.072+0000 7f3f6283cb80 20 client.26197 trim_cache size 0 max 16384 2023-07-14T11:14:47.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.089+0000 7f3f6283cb80 1 Processor -- start 2023-07-14T11:14:47.092 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.089+0000 7f3f6283cb80 1 -- start start 2023-07-14T11:14:47.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb73a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 0x5570afcc6b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afca3ec0 con 0x5570afcabbe0 2023-07-14T11:14:47.094 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afcc7e90 con 0x5570afcbaee0 2023-07-14T11:14:47.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afcac540 con 0x5570afcab2d0 2023-07-14T11:14:47.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37682/0 (socket says 172.21.15.40:37682) 2023-07-14T11:14:47.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2371110800 learned_addr learned my addr 172.21.15.40:0/2371110800 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:47.095 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb73a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 0x5570afcc6b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2371110800 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 msgr2=0x5570afcc6b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 0x5570afcc6b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2371110800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 msgr2=0x5570afcb73a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb73a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.090+0000 7f3f4cd9b700 1 -- 172.21.15.40:0/2371110800 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5570afa14c60 con 0x5570afcab2d0 2023-07-14T11:14:47.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.091+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 0x5570afcc6b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:47.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.091+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c00e970 tx=0x7f3f3c0d0b30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7577dde5cb349739 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.091+0000 7f3f337fe700 1 -- 172.21.15.40:0/2371110800 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c099db0 con 0x5570afcab2d0 2023-07-14T11:14:47.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.091+0000 7f3f337fe700 1 -- 172.21.15.40:0/2371110800 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f3c099940 con 0x5570afcab2d0 2023-07-14T11:14:47.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.091+0000 7f3f337fe700 1 -- 172.21.15.40:0/2371110800 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f3c0ca8e0 con 0x5570afcab2d0 2023-07-14T11:14:47.102 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2371110800 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 msgr2=0x5570afcb81e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c00e970 tx=0x7f3f3c0d0b30 comp rx=0 tx=0).stop 2023-07-14T11:14:47.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2371110800 shutdown_connections 2023-07-14T11:14:47.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcbaee0 0x5570afcc6b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcab2d0 0x5570afcb81e0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.103 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2371110800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcabbe0 0x5570afcb73a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2371110800 >> 172.21.15.40:0/2371110800 conn(0x5570afca1da0 msgr2=0x5570afca5e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2371110800 shutdown_connections 2023-07-14T11:14:47.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2371110800 wait complete. 2023-07-14T11:14:47.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.092+0000 7f3f6283cb80 1 -- start start 2023-07-14T11:14:47.104 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 Processor -- start 2023-07-14T11:14:47.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcbfa90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afd1d750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afcc0750 con 0x5570afcab2d0 2023-07-14T11:14:47.105 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afd1dc90 con 0x5570afcabbe0 2023-07-14T11:14:47.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.093+0000 7f3f6283cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5570afd1de00 con 0x5570afcbaee0 2023-07-14T11:14:47.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f6283cb80 10 client.0 _ll_register_callbacks cb 0x7f3f40004b40 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:14:47.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50606/0 (socket says 172.21.15.40:50606) 2023-07-14T11:14:47.106 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 -- 172.21.15.40:0/2266491120 learned_addr learned my addr 172.21.15.40:0/2266491120 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:14:47.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcbfa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afd1d750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 -- 172.21.15.40:0/2266491120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 msgr2=0x5570afd1d750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.107 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afd1d750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 -- 172.21.15.40:0/2266491120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 msgr2=0x5570afcbfa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:14:47.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcbfa90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 -- 172.21.15.40:0/2266491120 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5570afa2bde0 con 0x5570afcabbe0 2023-07-14T11:14:47.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f4d59c700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afd1d750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:14:47.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f47fff700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f3f34004550 tx=0x7f3f34003960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bad47e7e3e2a18e6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:14:47.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.094+0000 7f3f27fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:14:47.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3f34010840 con 0x5570afcabbe0 2023-07-14T11:14:47.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f3f34003e70 con 0x5570afcabbe0 2023-07-14T11:14:47.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3f3c099b20 con 0x5570afcabbe0 2023-07-14T11:14:47.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=35) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3f340109a0 con 0x5570afcabbe0 2023-07-14T11:14:47.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f6283cb80 10 client.26240 fetch_fsmap learned FSMap version 35 2023-07-14T11:14:47.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=35,monmap=4+}) v3 -- 0x5570afa52250 con 0x5570afcabbe0 2023-07-14T11:14:47.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.095+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3f3c099b20 con 0x5570afcabbe0 2023-07-14T11:14:47.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 35) v1 ==== 7510+0+0 (secure 0 0 0) 0x7f3f3401ccc0 con 0x5570afcabbe0 2023-07-14T11:14:47.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 10 client.26240 fetch_fsmap finished waiting for FSMap version 35 2023-07-14T11:14:47.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 10 client.26240 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:14:47.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 20 client.26240 populate_metadata read hostname 'smithi040' 2023-07-14T11:14:47.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afd1f670 0x5570afd21a80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:14:47.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 4 client.26240 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:14:47.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5570afbb9050 con 0x5570afd1f670 2023-07-14T11:14:47.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.096+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afd1f670 0x5570afd21a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:14:47.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f27fff700 10 client.26240 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:47.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f4cd9b700 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afd1f670 0x5570afd21a80 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=53c8dd9281736063 server_cookie=989f1b2788e57803 in_seq=0 out_seq=0 2023-07-14T11:14:47.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f27fff700 1 -- 172.21.15.40:0/2266491120 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x5570afbb9050 con 0x5570afd1f670 2023-07-14T11:14:47.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f27fff700 10 client.26240 handle_command_reply: tid=0 2023-07-14T11:14:47.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 client.26240 shutdown 2023-07-14T11:14:47.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 msgr2=0x5570afcbffd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f3f34004550 tx=0x7f3f34003960 comp rx=0 tx=0).stop 2023-07-14T11:14:47.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 shutdown_connections 2023-07-14T11:14:47.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5570afd1f670 0x5570afd21a80 crc :-1 s=READY pgs=82 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5570afcbaee0 0x5570afd1d750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5570afcab2d0 0x5570afcbfa90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 --2- 172.21.15.40:0/2266491120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5570afcabbe0 0x5570afcbffd0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f27fff700 0 client.26240 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:14:47.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 >> 172.21.15.40:0/2266491120 conn(0x5570afca1da0 msgr2=0x5570afcb2ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.097+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 shutdown_connections 2023-07-14T11:14:47.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f6283cb80 1 -- 172.21.15.40:0/2266491120 wait complete. 2023-07-14T11:14:47.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f6283cb80 20 client.26240 trim_cache size 0 max 16384 2023-07-14T11:14:47.117 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:14:47.118 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:14:47.118 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:14:47.118 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:14:47.118 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:14:47.119 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:14:47.119 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:14:47.119 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:14:47.120 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:14:47.120 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:14:47.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 msgr2=0x7f3f380a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 0x7f3f380a3770 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3f40009850 tx=0x7f3f40009780 comp rx=0 tx=0).stop 2023-07-14T11:14:47.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 msgr2=0x7f3f48195d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:14:47.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f3f3c00e9a0 tx=0x7f3f3c00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 shutdown_connections 2023-07-14T11:14:47.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3f380a12c0 0x7f3f380a3770 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3f48105af0 0x7f3f4819a6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3f480ffae0 0x7f3f48195d30 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 --2- 172.21.15.40:0/1435079613 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3f48100490 0x7f3f48196270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:14:47.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.098+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 >> 172.21.15.40:0/1435079613 conn(0x7f3f480fb5e0 msgr2=0x7f3f480fba90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:14:47.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.099+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 shutdown_connections 2023-07-14T11:14:47.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:14:47.099+0000 7f3f4efff700 1 -- 172.21.15.40:0/1435079613 wait complete. 2023-07-14T11:14:47.125 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:14:48.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:47 smithi130 ceph-mon[129405]: pgmap v82: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:47 smithi040 ceph-mon[97954]: pgmap v82: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:47 smithi045 ceph-mon[128791]: pgmap v82: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:50.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:49 smithi130 ceph-mon[129405]: pgmap v83: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:49 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:49 smithi040 ceph-mon[97954]: pgmap v83: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:49 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:50.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:49 smithi045 ceph-mon[128791]: pgmap v83: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 19 KiB/s rd, 6.4 MiB/s wr, 16 op/s 2023-07-14T11:14:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:49 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:14:52.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:51 smithi130 ceph-mon[129405]: pgmap v84: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 6.4 MiB/s wr, 21 op/s 2023-07-14T11:14:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:51 smithi040 ceph-mon[97954]: pgmap v84: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 6.4 MiB/s wr, 21 op/s 2023-07-14T11:14:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:51 smithi045 ceph-mon[128791]: pgmap v84: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 6.4 MiB/s wr, 21 op/s 2023-07-14T11:14:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:53 smithi130 ceph-mon[129405]: pgmap v85: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 8.3 KiB/s wr, 12 op/s 2023-07-14T11:14:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:53 smithi040 ceph-mon[97954]: pgmap v85: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 8.3 KiB/s wr, 12 op/s 2023-07-14T11:14:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:53 smithi045 ceph-mon[128791]: pgmap v85: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 8.3 KiB/s wr, 12 op/s 2023-07-14T11:14:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:55 smithi130 ceph-mon[129405]: pgmap v86: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 36 KiB/s wr, 20 op/s 2023-07-14T11:14:55.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:55 smithi040 ceph-mon[97954]: pgmap v86: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 36 KiB/s wr, 20 op/s 2023-07-14T11:14:55.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:55 smithi045 ceph-mon[128791]: pgmap v86: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 41 KiB/s rd, 36 KiB/s wr, 20 op/s 2023-07-14T11:14:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:57 smithi130 ceph-mon[129405]: pgmap v87: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:14:58.100 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:57 smithi045 ceph-mon[128791]: pgmap v87: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:14:58.100 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:57 smithi040 ceph-mon[97954]: pgmap v87: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:15:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:14:59 smithi130 ceph-mon[129405]: pgmap v88: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:15:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:14:59 smithi040 ceph-mon[97954]: pgmap v88: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:15:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:14:59 smithi045 ceph-mon[128791]: pgmap v88: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 34 KiB/s rd, 31 KiB/s wr, 13 op/s 2023-07-14T11:15:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:01 smithi130 ceph-mon[129405]: pgmap v89: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 33 KiB/s wr, 19 op/s 2023-07-14T11:15:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:01 smithi040 ceph-mon[97954]: pgmap v89: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 33 KiB/s wr, 19 op/s 2023-07-14T11:15:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:01 smithi045 ceph-mon[128791]: pgmap v89: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 37 KiB/s rd, 33 KiB/s wr, 19 op/s 2023-07-14T11:15:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:03 smithi130 ceph-mon[129405]: pgmap v90: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 KiB/s wr, 14 op/s 2023-07-14T11:15:04.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:03 smithi040 ceph-mon[97954]: pgmap v90: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 KiB/s wr, 14 op/s 2023-07-14T11:15:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:03 smithi045 ceph-mon[128791]: pgmap v90: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 33 KiB/s rd, 31 KiB/s wr, 14 op/s 2023-07-14T11:15:05.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:05 smithi130 ceph-mon[129405]: pgmap v91: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 58 KiB/s wr, 33 op/s 2023-07-14T11:15:05.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:05 smithi040 ceph-mon[97954]: pgmap v91: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 58 KiB/s wr, 33 op/s 2023-07-14T11:15:05.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:05 smithi045 ceph-mon[128791]: pgmap v91: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 58 KiB/s wr, 33 op/s 2023-07-14T11:15:08.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:07 smithi130 ceph-mon[129405]: pgmap v92: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:08.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:07 smithi040 ceph-mon[97954]: pgmap v92: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:08.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:07 smithi045 ceph-mon[128791]: pgmap v92: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:09 smithi130 ceph-mon[129405]: pgmap v93: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:09 smithi040 ceph-mon[97954]: pgmap v93: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:09 smithi045 ceph-mon[128791]: pgmap v93: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 29 KiB/s wr, 25 op/s 2023-07-14T11:15:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:11 smithi130 ceph-mon[129405]: pgmap v94: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 35 KiB/s wr, 31 op/s 2023-07-14T11:15:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:11 smithi040 ceph-mon[97954]: pgmap v94: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 35 KiB/s wr, 31 op/s 2023-07-14T11:15:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:11 smithi045 ceph-mon[128791]: pgmap v94: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 25 KiB/s rd, 35 KiB/s wr, 31 op/s 2023-07-14T11:15:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:13 smithi130 ceph-mon[129405]: pgmap v95: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 32 KiB/s wr, 24 op/s 2023-07-14T11:15:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:13 smithi040 ceph-mon[97954]: pgmap v95: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 32 KiB/s wr, 24 op/s 2023-07-14T11:15:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:13 smithi045 ceph-mon[128791]: pgmap v95: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 21 KiB/s rd, 32 KiB/s wr, 24 op/s 2023-07-14T11:15:15.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:15 smithi130 ceph-mon[129405]: pgmap v96: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 36 KiB/s wr, 30 op/s 2023-07-14T11:15:15.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:15 smithi040 ceph-mon[97954]: pgmap v96: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 36 KiB/s wr, 30 op/s 2023-07-14T11:15:15.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:15 smithi045 ceph-mon[128791]: pgmap v96: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 27 KiB/s rd, 36 KiB/s wr, 30 op/s 2023-07-14T11:15:17.126 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:15:17.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 -- 172.21.15.40:0/2612070235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 msgr2=0x7f58940febf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 --2- 172.21.15.40:0/2612070235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f58940febf0 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7f5888009990 tx=0x7f5888009ca0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.362 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 -- 172.21.15.40:0/2612070235 shutdown_connections 2023-07-14T11:15:17.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 --2- 172.21.15.40:0/2612070235 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f589410c3d0 0x7f589410e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 --2- 172.21.15.40:0/2612070235 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f58940febf0 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 --2- 172.21.15.40:0/2612070235 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f58940ff1c0 0x7f589410be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 -- 172.21.15.40:0/2612070235 >> 172.21.15.40:0/2612070235 conn(0x7f58940fa4b0 msgr2=0x7f58940fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.363 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 -- 172.21.15.40:0/2612070235 shutdown_connections 2023-07-14T11:15:17.364 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.360+0000 7f589abff700 1 -- 172.21.15.40:0/2612070235 wait complete. 2023-07-14T11:15:17.365 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.361+0000 7f589abff700 1 Processor -- start 2023-07-14T11:15:17.365 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 -- start start 2023-07-14T11:15:17.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58940ff1c0 0x7f58941982f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 0x7f589419c750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f589419cd90 con 0x7f58940ff1c0 2023-07-14T11:15:17.366 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f589419cf00 con 0x7f589410c3d0 2023-07-14T11:15:17.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589abff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f589419d0c0 con 0x7f58940fe820 2023-07-14T11:15:17.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589899b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589899b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:43584/0 (socket says 172.21.15.40:43584) 2023-07-14T11:15:17.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.362+0000 7f589899b700 1 -- 172.21.15.40:0/2532208197 learned_addr learned my addr 172.21.15.40:0/2532208197 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:17.367 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589919c700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 0x7f589419c750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f5893fff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58940ff1c0 0x7f58941982f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 -- 172.21.15.40:0/2532208197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 msgr2=0x7f589419c750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 0x7f589419c750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 -- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58940ff1c0 msgr2=0x7f58941982f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.368 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58940ff1c0 0x7f58941982f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 -- 172.21.15.40:0/2532208197 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58880095f0 con 0x7f58940fe820 2023-07-14T11:15:17.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589919c700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 0x7f589419c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:15:17.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f589899b700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f5888000c00 tx=0x7f5888005ef0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.363+0000 7f5891ffb700 1 -- 172.21.15.40:0/2532208197 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f588801d070 con 0x7f58940fe820 2023-07-14T11:15:17.369 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.364+0000 7f5891ffb700 1 -- 172.21.15.40:0/2532208197 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f588800fbc0 con 0x7f58940fe820 2023-07-14T11:15:17.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.364+0000 7f5891ffb700 1 -- 172.21.15.40:0/2532208197 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58880173f0 con 0x7f58940fe820 2023-07-14T11:15:17.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.364+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f589419d340 con 0x7f58940fe820 2023-07-14T11:15:17.370 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.364+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58941a4be0 con 0x7f58940fe820 2023-07-14T11:15:17.371 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.369+0000 7f5891ffb700 1 -- 172.21.15.40:0/2532208197 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f5888021410 con 0x7f58940fe820 2023-07-14T11:15:17.371 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.370+0000 7f58ae5b8b80 1 Processor -- start 2023-07-14T11:15:17.372 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.370+0000 7f58ae5b8b80 1 -- start start 2023-07-14T11:15:17.372 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.370+0000 7f5891ffb700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 0x7f58840a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.373 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.373 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f5891ffb700 1 -- 172.21.15.40:0/2532208197 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f58880ce360 con 0x7f58940fe820 2023-07-14T11:15:17.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff5590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff7eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cef0cb0 con 0x55ff2cff5ad0 2023-07-14T11:15:17.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cf43a40 con 0x55ff2cfeabe0 2023-07-14T11:15:17.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cf3ced0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589919c700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 0x7f58840a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:43588/0 (socket says 172.21.15.40:43588) 2023-07-14T11:15:17.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 -- 172.21.15.40:0/256732672 learned_addr learned my addr 172.21.15.40:0/256732672 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:17.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f5893fff700 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff5590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589919c700 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff7eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 -- 172.21.15.40:0/256732672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfeabe0 msgr2=0x55ff2cff5590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff5590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 -- 172.21.15.40:0/256732672 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cff5ad0 msgr2=0x55ff2cff7eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff7eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.371+0000 7f589899b700 1 -- 172.21.15.40:0/256732672 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff2cd6ade0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.372+0000 7f589899b700 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f58880a2fe0 tx=0x7f5888009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d16dec1a499b3679 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.379 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.372+0000 7f589919c700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 0x7f58840a37f0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f5894069c40 tx=0x7f588c010c50 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.372+0000 7f587effd700 1 -- 172.21.15.40:0/256732672 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f588809f180 con 0x55ff2cfea2d0 2023-07-14T11:15:17.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.372+0000 7f587effd700 1 -- 172.21.15.40:0/256732672 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f58880cbe90 con 0x55ff2cfea2d0 2023-07-14T11:15:17.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.372+0000 7f587effd700 1 -- 172.21.15.40:0/256732672 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f588809a8a0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/256732672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 msgr2=0x55ff2cfea6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f58880a2fe0 tx=0x7f5888009ca0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/256732672 shutdown_connections 2023-07-14T11:15:17.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff7eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfea2d0 0x55ff2cfea6a0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/256732672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff5590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/256732672 >> 172.21.15.40:0/256732672 conn(0x55ff2cfe0da0 msgr2=0x55ff2cfe66a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/256732672 shutdown_connections 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/256732672 wait complete. 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.373+0000 7f58ae5b8b80 1 -- start start 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 Processor -- start 2023-07-14T11:15:17.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff5ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff6920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cedcac0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cedc7c0 con 0x55ff2cff5ad0 2023-07-14T11:15:17.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cdc0920 con 0x55ff2cfeabe0 2023-07-14T11:15:17.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f58ae5b8b80 10 client.0 _ll_register_callbacks cb 0x55ff2ceda760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:15:17.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f5893fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.374+0000 7f5893fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:43604/0 (socket says 172.21.15.40:43604) 2023-07-14T11:15:17.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 -- 172.21.15.40:0/3315937856 learned_addr learned my addr 172.21.15.40:0/3315937856 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:17.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f589919c700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff6920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f589899b700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff5ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 -- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cff5ad0 msgr2=0x55ff2cff6920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff6920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 -- 172.21.15.40:0/3315937856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 msgr2=0x55ff2cff5ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff5ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 -- 172.21.15.40:0/3315937856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58941a4be0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f589899b700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff5ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:15:17.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5893fff700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f5880012700 tx=0x7f5880012a10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e167d04bc79c91ab server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5873fff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:15:17.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5880017070 con 0x55ff2cfeabe0 2023-07-14T11:15:17.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5880004260 con 0x55ff2cfeabe0 2023-07-14T11:15:17.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.375+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ff2ce964f0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=36) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5880004420 con 0x55ff2cfeabe0 2023-07-14T11:15:17.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 10 client.26215 fetch_fsmap learned FSMap version 36 2023-07-14T11:15:17.387 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=36,monmap=4+}) v3 -- 0x55ff2cd53c60 con 0x55ff2cfeabe0 2023-07-14T11:15:17.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ff2ce964f0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 36) v1 ==== 7506+0+0 (secure 0 0 0) 0x7f588000f6f0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 10 client.26215 fetch_fsmap finished waiting for FSMap version 36 2023-07-14T11:15:17.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 10 client.26215 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:15:17.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 20 client.26215 populate_metadata read hostname 'smithi040' 2023-07-14T11:15:17.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d0050a0 0x55ff2d0074b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 4 client.26215 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:15:17.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.376+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ff2cf35eb0 con 0x55ff2d0050a0 2023-07-14T11:15:17.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.377+0000 7f589899b700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d0050a0 0x55ff2d0074b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.377+0000 7f589899b700 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d0050a0 0x55ff2d0074b0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1995c7c20a845ff9 server_cookie=d855dc1078717caa in_seq=0 out_seq=0 2023-07-14T11:15:17.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.377+0000 7f5873fff700 10 client.26215 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:17.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.378+0000 7f5873fff700 1 -- 172.21.15.40:0/3315937856 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55ff2cf35eb0 con 0x55ff2d0050a0 2023-07-14T11:15:17.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.378+0000 7f5873fff700 10 client.26215 handle_command_reply: tid=0 2023-07-14T11:15:17.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.378+0000 7f58ae5b8b80 1 client.26215 shutdown 2023-07-14T11:15:17.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.378+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 msgr2=0x55ff2cff63e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.391 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.378+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7f5880012700 tx=0x7f5880012a10 comp rx=0 tx=0).stop 2023-07-14T11:15:17.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 shutdown_connections 2023-07-14T11:15:17.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d0050a0 0x55ff2d0074b0 crc :-1 s=READY pgs=83 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cff5ad0 0x55ff2cff6920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff5ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3315937856 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff63e0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f5873fff700 0 client.26215 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:17.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 >> 172.21.15.40:0/3315937856 conn(0x55ff2cfe0da0 msgr2=0x55ff2cfe6230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 shutdown_connections 2023-07-14T11:15:17.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3315937856 wait complete. 2023-07-14T11:15:17.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.379+0000 7f58ae5b8b80 20 client.26215 trim_cache size 0 max 16384 2023-07-14T11:15:17.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.397+0000 7f58ae5b8b80 1 Processor -- start 2023-07-14T11:15:17.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 -- start start 2023-07-14T11:15:17.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff4360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d001430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cfe09e0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2d0027c0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cfec1d0 con 0x55ff2cff5ad0 2023-07-14T11:15:17.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f589899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f589899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:52056/0 (socket says 172.21.15.40:52056) 2023-07-14T11:15:17.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f5893fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff4360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f589899b700 1 -- 172.21.15.40:0/3389469102 learned_addr learned my addr 172.21.15.40:0/3389469102 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:17.402 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.398+0000 7f589919c700 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d001430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 -- 172.21.15.40:0/3389469102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 msgr2=0x55ff2d001430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d001430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 -- 172.21.15.40:0/3389469102 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 msgr2=0x55ff2cff4360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff4360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.403 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 -- 172.21.15.40:0/3389469102 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ff2cd91250 con 0x55ff2cfea2d0 2023-07-14T11:15:17.404 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f589899b700 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f588809ae50 tx=0x7f58880059d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=348d366b73ad426 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.406 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f587f7fe700 1 -- 172.21.15.40:0/3389469102 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f58880ce040 con 0x55ff2cfea2d0 2023-07-14T11:15:17.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f587f7fe700 1 -- 172.21.15.40:0/3389469102 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f58880ce1a0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.399+0000 7f587f7fe700 1 -- 172.21.15.40:0/3389469102 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f588809b610 con 0x55ff2cfea2d0 2023-07-14T11:15:17.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3389469102 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 msgr2=0x55ff2cff9ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7f588809ae50 tx=0x7f58880059d0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.407 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3389469102 shutdown_connections 2023-07-14T11:15:17.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d001430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff9ed0 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/3389469102 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff4360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3389469102 >> 172.21.15.40:0/3389469102 conn(0x55ff2cfe0da0 msgr2=0x55ff2cfef440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3389469102 shutdown_connections 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.400+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/3389469102 wait complete. 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.401+0000 7f58ae5b8b80 1 -- start start 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.401+0000 7f58ae5b8b80 1 Processor -- start 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.401+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff8120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d057f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2cff88c0 con 0x55ff2cfeabe0 2023-07-14T11:15:17.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2d0584f0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ff2d0586b0 con 0x55ff2cff5ad0 2023-07-14T11:15:17.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f58ae5b8b80 10 client.0 _ll_register_callbacks cb 0x7f5880009c60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:15:17.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:52064/0 (socket says 172.21.15.40:52064) 2023-07-14T11:15:17.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 -- 172.21.15.40:0/361087148 learned_addr learned my addr 172.21.15.40:0/361087148 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:17.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f5893fff700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff8120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589919c700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d057f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 -- 172.21.15.40:0/361087148 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 msgr2=0x55ff2d057f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d057f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 -- 172.21.15.40:0/361087148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 msgr2=0x55ff2cff8120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:17.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff8120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.402+0000 7f589899b700 1 -- 172.21.15.40:0/361087148 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ff2cd6ade0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f589899b700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f58880994a0 tx=0x7f588800b4b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=32431bee5eb1d54c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:17.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f5873fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:15:17.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f588809a640 con 0x55ff2cfea2d0 2023-07-14T11:15:17.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f58880b92c0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f58880cbe90 con 0x55ff2cfea2d0 2023-07-14T11:15:17.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=36) v2 ==== 24+0+0 (secure 0 0 0) 0x7f58880b9480 con 0x55ff2cfea2d0 2023-07-14T11:15:17.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f58ae5b8b80 10 client.26258 fetch_fsmap learned FSMap version 36 2023-07-14T11:15:17.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=36,monmap=4+}) v3 -- 0x55ff2cd692b0 con 0x55ff2cfea2d0 2023-07-14T11:15:17.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.403+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f58880cbe90 con 0x55ff2cfea2d0 2023-07-14T11:15:17.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 36) v1 ==== 7506+0+0 (secure 0 0 0) 0x7f58880a3180 con 0x55ff2cfea2d0 2023-07-14T11:15:17.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 10 client.26258 fetch_fsmap finished waiting for FSMap version 36 2023-07-14T11:15:17.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 10 client.26258 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:15:17.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 20 client.26258 populate_metadata read hostname 'smithi040' 2023-07-14T11:15:17.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d059910 0x55ff2d05bcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:17.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 4 client.26258 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:15:17.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ff2cef8050 con 0x55ff2d059910 2023-07-14T11:15:17.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f5893fff700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d059910 0x55ff2d05bcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:17.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f5873fff700 10 client.26258 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:17.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.404+0000 7f5893fff700 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d059910 0x55ff2d05bcf0 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d96b62734c6eff65 server_cookie=177b542ce5b9b5bf in_seq=0 out_seq=0 2023-07-14T11:15:17.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f5873fff700 1 -- 172.21.15.40:0/361087148 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55ff2cef8050 con 0x55ff2d059910 2023-07-14T11:15:17.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f5873fff700 10 client.26258 handle_command_reply: tid=0 2023-07-14T11:15:17.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 client.26258 shutdown 2023-07-14T11:15:17.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 msgr2=0x55ff2cff7be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f58880994a0 tx=0x7f588800b4b0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 shutdown_connections 2023-07-14T11:15:17.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ff2d059910 0x55ff2d05bcf0 crc :-1 s=READY pgs=84 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ff2cff5ad0 0x55ff2d057f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ff2cfea2d0 0x55ff2cff7be0 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f5873fff700 0 client.26258 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:17.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 --2- 172.21.15.40:0/361087148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ff2cfeabe0 0x55ff2cff8120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 >> 172.21.15.40:0/361087148 conn(0x55ff2cfe0da0 msgr2=0x55ff2cff4360 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 shutdown_connections 2023-07-14T11:15:17.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.405+0000 7f58ae5b8b80 1 -- 172.21.15.40:0/361087148 wait complete. 2023-07-14T11:15:17.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f58ae5b8b80 20 client.26258 trim_cache size 0 max 16384 2023-07-14T11:15:17.420 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:15:17.420 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:15:17.420 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:15:17.421 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:15:17.421 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:15:17.421 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:15:17.421 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:15:17.421 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:15:17.422 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:15:17.422 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:15:17.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 msgr2=0x7f58840a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 0x7f58840a37f0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f5894069c40 tx=0x7f588c010c50 comp rx=0 tx=0).stop 2023-07-14T11:15:17.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 msgr2=0x7f5894197db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:17.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f5888000c00 tx=0x7f5888005ef0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 shutdown_connections 2023-07-14T11:15:17.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f58840a1340 0x7f58840a37f0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f589410c3d0 0x7f589419c750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f58940fe820 0x7f5894197db0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 --2- 172.21.15.40:0/2532208197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58940ff1c0 0x7f58941982f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:17.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.406+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 >> 172.21.15.40:0/2532208197 conn(0x7f58940fa4b0 msgr2=0x7f589410d6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:17.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.407+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 shutdown_connections 2023-07-14T11:15:17.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:17.407+0000 7f589abff700 1 -- 172.21.15.40:0/2532208197 wait complete. 2023-07-14T11:15:17.426 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:15:18.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:17 smithi130 ceph-mon[129405]: pgmap v97: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:17 smithi040 ceph-mon[97954]: pgmap v97: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:17 smithi045 ceph-mon[128791]: pgmap v97: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:19 smithi130 ceph-mon[129405]: pgmap v98: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:19 smithi040 ceph-mon[97954]: pgmap v98: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:19 smithi045 ceph-mon[128791]: pgmap v98: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 9.6 KiB/s wr, 11 op/s 2023-07-14T11:15:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:21.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:20 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:15:21.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:20 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:15:21.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:20 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:15:22.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:21 smithi130 ceph-mon[129405]: pgmap v99: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 KiB/s wr, 17 op/s 2023-07-14T11:15:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:21 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:15:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:21 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:15:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:21 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:15:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:21 smithi040 ceph-mon[97954]: pgmap v99: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 KiB/s wr, 17 op/s 2023-07-14T11:15:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:21 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:15:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:21 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:15:22.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:21 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:15:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:21 smithi045 ceph-mon[128791]: pgmap v99: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 KiB/s rd, 12 KiB/s wr, 17 op/s 2023-07-14T11:15:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:21 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:15:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:21 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:15:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:21 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:15:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:23 smithi130 ceph-mon[129405]: pgmap v100: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.4 KiB/s wr, 11 op/s 2023-07-14T11:15:24.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:23 smithi040 ceph-mon[97954]: pgmap v100: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.4 KiB/s wr, 11 op/s 2023-07-14T11:15:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:23 smithi045 ceph-mon[128791]: pgmap v100: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.4 KiB/s wr, 11 op/s 2023-07-14T11:15:25.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:25 smithi130 ceph-mon[129405]: pgmap v101: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 7.8 KiB/s wr, 16 op/s 2023-07-14T11:15:25.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:25 smithi040 ceph-mon[97954]: pgmap v101: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 7.8 KiB/s wr, 16 op/s 2023-07-14T11:15:25.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:25 smithi045 ceph-mon[128791]: pgmap v101: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 7.8 KiB/s wr, 16 op/s 2023-07-14T11:15:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:27 smithi130 ceph-mon[129405]: pgmap v102: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:27 smithi040 ceph-mon[97954]: pgmap v102: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:27 smithi045 ceph-mon[128791]: pgmap v102: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:29 smithi130 ceph-mon[129405]: pgmap v103: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:30.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:29 smithi040 ceph-mon[97954]: pgmap v103: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:29 smithi045 ceph-mon[128791]: pgmap v103: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 7.3 KiB/s rd, 3.4 KiB/s wr, 10 op/s 2023-07-14T11:15:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:31 smithi130 ceph-mon[129405]: pgmap v104: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 9.5 KiB/s wr, 16 op/s 2023-07-14T11:15:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:31 smithi040 ceph-mon[97954]: pgmap v104: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 9.5 KiB/s wr, 16 op/s 2023-07-14T11:15:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:31 smithi045 ceph-mon[128791]: pgmap v104: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 9.5 KiB/s wr, 16 op/s 2023-07-14T11:15:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:33 smithi130 ceph-mon[129405]: pgmap v105: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.5 KiB/s wr, 11 op/s 2023-07-14T11:15:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:33 smithi040 ceph-mon[97954]: pgmap v105: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.5 KiB/s wr, 11 op/s 2023-07-14T11:15:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:33 smithi045 ceph-mon[128791]: pgmap v105: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 7.5 KiB/s wr, 11 op/s 2023-07-14T11:15:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:35 smithi130 ceph-mon[129405]: pgmap v106: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 8.2 KiB/s wr, 16 op/s 2023-07-14T11:15:35.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:35 smithi040 ceph-mon[97954]: pgmap v106: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 8.2 KiB/s wr, 16 op/s 2023-07-14T11:15:35.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:35 smithi045 ceph-mon[128791]: pgmap v106: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 8.2 KiB/s wr, 16 op/s 2023-07-14T11:15:38.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:37 smithi130 ceph-mon[129405]: pgmap v107: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:38.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:37 smithi040 ceph-mon[97954]: pgmap v107: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:38.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:37 smithi045 ceph-mon[128791]: pgmap v107: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:40.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:39 smithi130 ceph-mon[129405]: pgmap v108: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:39 smithi040 ceph-mon[97954]: pgmap v108: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:40.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:39 smithi045 ceph-mon[128791]: pgmap v108: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 KiB/s rd, 6.8 KiB/s wr, 11 op/s 2023-07-14T11:15:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:41 smithi130 ceph-mon[129405]: pgmap v109: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 10 KiB/s wr, 16 op/s 2023-07-14T11:15:42.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:41 smithi040 ceph-mon[97954]: pgmap v109: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 10 KiB/s wr, 16 op/s 2023-07-14T11:15:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:41 smithi045 ceph-mon[128791]: pgmap v109: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 10 KiB/s wr, 16 op/s 2023-07-14T11:15:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:43 smithi130 ceph-mon[129405]: pgmap v110: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.7 KiB/s rd, 4.4 KiB/s wr, 10 op/s 2023-07-14T11:15:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:43 smithi040 ceph-mon[97954]: pgmap v110: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.7 KiB/s rd, 4.4 KiB/s wr, 10 op/s 2023-07-14T11:15:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:43 smithi045 ceph-mon[128791]: pgmap v110: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 8.7 KiB/s rd, 4.4 KiB/s wr, 10 op/s 2023-07-14T11:15:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:45 smithi130 ceph-mon[129405]: pgmap v111: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 11 KiB/s wr, 17 op/s 2023-07-14T11:15:45.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:45 smithi040 ceph-mon[97954]: pgmap v111: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 11 KiB/s wr, 17 op/s 2023-07-14T11:15:45.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:45 smithi045 ceph-mon[128791]: pgmap v111: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 11 KiB/s wr, 17 op/s 2023-07-14T11:15:47.427 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:15:47.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 -- 172.21.15.40:0/1996620433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c10a040 msgr2=0x7fd67c10c4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 --2- 172.21.15.40:0/1996620433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c10a040 0x7fd67c10c4b0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7fd67400b0a0 tx=0x7fd67400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 -- 172.21.15.40:0/1996620433 shutdown_connections 2023-07-14T11:15:47.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 --2- 172.21.15.40:0/1996620433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c10a040 0x7fd67c10c4b0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 --2- 172.21.15.40:0/1996620433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c100d00 0x7fd67c1010d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.670 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 --2- 172.21.15.40:0/1996620433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c109b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 -- 172.21.15.40:0/1996620433 >> 172.21.15.40:0/1996620433 conn(0x7fd67c0fa4b0 msgr2=0x7fd67c0fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 -- 172.21.15.40:0/1996620433 shutdown_connections 2023-07-14T11:15:47.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.666+0000 7fd6847ff700 1 -- 172.21.15.40:0/1996620433 wait complete. 2023-07-14T11:15:47.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.668+0000 7fd6847ff700 1 Processor -- start 2023-07-14T11:15:47.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.668+0000 7fd6847ff700 1 -- start start 2023-07-14T11:15:47.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c10a040 0x7fd67c19c890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd67c19ced0 con 0x7fd67c10a040 2023-07-14T11:15:47.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd67c19d040 con 0x7fd67c100d00 2023-07-14T11:15:47.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd6847ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd67c19d200 con 0x7fd67c101610 2023-07-14T11:15:47.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd681d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd68259b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd68259b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48296/0 (socket says 172.21.15.40:48296) 2023-07-14T11:15:47.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd681d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59034/0 (socket says 172.21.15.40:59034) 2023-07-14T11:15:47.674 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.669+0000 7fd68259b700 1 -- 172.21.15.40:0/2146790540 learned_addr learned my addr 172.21.15.40:0/2146790540 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:47.675 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd682d9c700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c10a040 0x7fd67c19c890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 -- 172.21.15.40:0/2146790540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 msgr2=0x7fd67c197ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 -- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c10a040 msgr2=0x7fd67c19c890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c10a040 0x7fd67c19c890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.676 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 -- 172.21.15.40:0/2146790540 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd678009520 con 0x7fd67c101610 2023-07-14T11:15:47.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd68259b700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:15:47.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd681d9a700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd66c00e9f0 tx=0x7fd66c00ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd6737fe700 1 -- 172.21.15.40:0/2146790540 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd66c004220 con 0x7fd67c101610 2023-07-14T11:15:47.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd6737fe700 1 -- 172.21.15.40:0/2146790540 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd66c004cf0 con 0x7fd67c101610 2023-07-14T11:15:47.677 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.670+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd674009d00 con 0x7fd67c101610 2023-07-14T11:15:47.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.671+0000 7fd6737fe700 1 -- 172.21.15.40:0/2146790540 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd66c005020 con 0x7fd67c101610 2023-07-14T11:15:47.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.671+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd67c1a4b40 con 0x7fd67c101610 2023-07-14T11:15:47.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.674+0000 7fd6737fe700 1 -- 172.21.15.40:0/2146790540 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fd66c004750 con 0x7fd67c101610 2023-07-14T11:15:47.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.675+0000 7fd6980cfb80 1 Processor -- start 2023-07-14T11:15:47.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.675+0000 7fd6737fe700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 0x7fd6680a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.675+0000 7fd6980cfb80 1 -- start start 2023-07-14T11:15:47.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6737fe700 1 -- 172.21.15.40:0/2146790540 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd66c014070 con 0x7fd67c101610 2023-07-14T11:15:47.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd682d9c700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 0x7fd6680a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa0590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647fa0ad0 0x55c647fa2eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647e9bcb0 con 0x55c647fa0ad0 2023-07-14T11:15:47.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647eeea40 con 0x55c647f95be0 2023-07-14T11:15:47.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647ee7ed0 con 0x55c647f952d0 2023-07-14T11:15:47.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59046/0 (socket says 172.21.15.40:59046) 2023-07-14T11:15:47.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd68259b700 1 -- 172.21.15.40:0/664537113 learned_addr learned my addr 172.21.15.40:0/664537113 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:47.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.676+0000 7fd681d9a700 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa0590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 -- 172.21.15.40:0/664537113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 msgr2=0x55c647fa0590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa0590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 -- 172.21.15.40:0/664537113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647fa0ad0 msgr2=0x55c647fa2eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647fa0ad0 0x55c647fa2eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 -- 172.21.15.40:0/664537113 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c647d15de0 con 0x55c647f952d0 2023-07-14T11:15:47.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd681d9a700 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa0590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:15:47.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd682d9c700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 0x7fd6680a3770 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fd674009fd0 tx=0x7fd6740094c0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd68259b700 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fd678011840 tx=0x7fd678011b50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=352c9a3531cf72d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd670ff9700 1 -- 172.21.15.40:0/664537113 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd678011030 con 0x55c647f952d0 2023-07-14T11:15:47.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd670ff9700 1 -- 172.21.15.40:0/664537113 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd678011190 con 0x55c647f952d0 2023-07-14T11:15:47.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.677+0000 7fd670ff9700 1 -- 172.21.15.40:0/664537113 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd678010890 con 0x55c647f952d0 2023-07-14T11:15:47.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 -- 172.21.15.40:0/664537113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 msgr2=0x55c647f956a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7fd678011840 tx=0x7fd678011b50 comp rx=0 tx=0).stop 2023-07-14T11:15:47.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 -- 172.21.15.40:0/664537113 shutdown_connections 2023-07-14T11:15:47.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647fa0ad0 0x55c647fa2eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647f956a0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/664537113 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa0590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.678+0000 7fd6980cfb80 1 -- 172.21.15.40:0/664537113 >> 172.21.15.40:0/664537113 conn(0x55c647f8bda0 msgr2=0x55c647f916a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.679+0000 7fd6980cfb80 1 -- 172.21.15.40:0/664537113 shutdown_connections 2023-07-14T11:15:47.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.679+0000 7fd6980cfb80 1 -- 172.21.15.40:0/664537113 wait complete. 2023-07-14T11:15:47.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.679+0000 7fd6980cfb80 1 -- start start 2023-07-14T11:15:47.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 Processor -- start 2023-07-14T11:15:47.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9d6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9dcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647e87ac0 con 0x55c647f95be0 2023-07-14T11:15:47.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647e877c0 con 0x55c647fa0ad0 2023-07-14T11:15:47.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.680+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647d6b920 con 0x55c647f952d0 2023-07-14T11:15:47.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd6980cfb80 10 client.0 _ll_register_callbacks cb 0x55c647e85760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:15:47.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59056/0 (socket says 172.21.15.40:59056) 2023-07-14T11:15:47.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 -- 172.21.15.40:0/3389747766 learned_addr learned my addr 172.21.15.40:0/3389747766 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:47.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd682d9c700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9dcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd681d9a700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9d6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 -- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 msgr2=0x55c647f9dcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9dcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 -- 172.21.15.40:0/3389747766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 msgr2=0x55c647f9d6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9d6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 -- 172.21.15.40:0/3389747766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd67c1a4b40 con 0x55c647f952d0 2023-07-14T11:15:47.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd681d9a700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:15:47.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd665ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:15:47.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.681+0000 7fd68259b700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fd6780056b0 tx=0x7fd678005790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b4f20c622a50811f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd678027070 con 0x55c647f952d0 2023-07-14T11:15:47.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd67800f770 con 0x55c647f952d0 2023-07-14T11:15:47.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c647e414f0 con 0x55c647f952d0 2023-07-14T11:15:47.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd678011300 con 0x55c647f952d0 2023-07-14T11:15:47.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd6980cfb80 10 client.26233 fetch_fsmap learned FSMap version 37 2023-07-14T11:15:47.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) v3 -- 0x7fd6780114a0 con 0x55c647f952d0 2023-07-14T11:15:47.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.682+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c647e414f0 con 0x55c647f952d0 2023-07-14T11:15:47.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 37) v1 ==== 7502+0+0 (secure 0 0 0) 0x7fd67802a030 con 0x55c647f952d0 2023-07-14T11:15:47.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 10 client.26233 fetch_fsmap finished waiting for FSMap version 37 2023-07-14T11:15:47.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 10 client.26233 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:15:47.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 20 client.26233 populate_metadata read hostname 'smithi040' 2023-07-14T11:15:47.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c647fb02d0 0x55c647fb26e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 4 client.26233 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:15:47.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c647ee0eb0 con 0x55c647fb02d0 2023-07-14T11:15:47.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd681d9a700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c647fb02d0 0x55c647fb26e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd681d9a700 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c647fb02d0 0x55c647fb26e0 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=a4a747e6b2287236 server_cookie=125d9dbab6c69d8c in_seq=0 out_seq=0 2023-07-14T11:15:47.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.683+0000 7fd665ffb700 10 client.26233 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:47.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.684+0000 7fd665ffb700 1 -- 172.21.15.40:0/3389747766 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c647ee0eb0 con 0x55c647fb02d0 2023-07-14T11:15:47.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.684+0000 7fd665ffb700 10 client.26233 handle_command_reply: tid=0 2023-07-14T11:15:47.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.684+0000 7fd6980cfb80 1 client.26233 shutdown 2023-07-14T11:15:47.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 msgr2=0x55c647fa0460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7fd6780056b0 tx=0x7fd678005790 comp rx=0 tx=0).stop 2023-07-14T11:15:47.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 shutdown_connections 2023-07-14T11:15:47.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c647fb02d0 0x55c647fb26e0 crc :-1 s=READY pgs=85 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9dcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0460 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/3389747766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd665ffb700 0 client.26233 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:47.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 >> 172.21.15.40:0/3389747766 conn(0x55c647f8bda0 msgr2=0x55c647f91290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 shutdown_connections 2023-07-14T11:15:47.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 1 -- 172.21.15.40:0/3389747766 wait complete. 2023-07-14T11:15:47.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.685+0000 7fd6980cfb80 20 client.26233 trim_cache size 0 max 16384 2023-07-14T11:15:47.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.720+0000 7fd6980cfb80 1 Processor -- start 2023-07-14T11:15:47.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.720+0000 7fd6980cfb80 1 -- start start 2023-07-14T11:15:47.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9cef0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.724 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9d430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647f96540 con 0x55c647f95be0 2023-07-14T11:15:47.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647f9ecc0 con 0x55c647fa0ad0 2023-07-14T11:15:47.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647f95fb0 con 0x55c647f952d0 2023-07-14T11:15:47.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd68259b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59064/0 (socket says 172.21.15.40:59064) 2023-07-14T11:15:47.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd68259b700 1 -- 172.21.15.40:0/2051818902 learned_addr learned my addr 172.21.15.40:0/2051818902 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:47.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd682d9c700 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9d430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.721+0000 7fd681d9a700 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9cef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 -- 172.21.15.40:0/2051818902 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 msgr2=0x55c647f9d430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9d430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 -- 172.21.15.40:0/2051818902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 msgr2=0x55c647f9cef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9cef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 -- 172.21.15.40:0/2051818902 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c647cfec60 con 0x55c647f952d0 2023-07-14T11:15:47.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd68259b700 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fd678011b90 tx=0x7fd678011ee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=15ab2f055c8bbff8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd6717fa700 1 -- 172.21.15.40:0/2051818902 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd678027070 con 0x55c647f952d0 2023-07-14T11:15:47.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd6717fa700 1 -- 172.21.15.40:0/2051818902 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd678011300 con 0x55c647f952d0 2023-07-14T11:15:47.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.722+0000 7fd6717fa700 1 -- 172.21.15.40:0/2051818902 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd678010760 con 0x55c647f952d0 2023-07-14T11:15:47.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 -- 172.21.15.40:0/2051818902 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 msgr2=0x55c647fa0000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fd678011b90 tx=0x7fd678011ee0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 -- 172.21.15.40:0/2051818902 shutdown_connections 2023-07-14T11:15:47.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647fa0ad0 0x55c647f9d430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.730 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647f952d0 0x55c647fa0000 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/2051818902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f95be0 0x55c647f9cef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 -- 172.21.15.40:0/2051818902 >> 172.21.15.40:0/2051818902 conn(0x55c647f8bda0 msgr2=0x55c647fac6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 -- 172.21.15.40:0/2051818902 shutdown_connections 2023-07-14T11:15:47.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.723+0000 7fd6980cfb80 1 -- 172.21.15.40:0/2051818902 wait complete. 2023-07-14T11:15:47.731 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.724+0000 7fd6980cfb80 1 -- start start 2023-07-14T11:15:47.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.724+0000 7fd6980cfb80 1 Processor -- start 2023-07-14T11:15:47.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f952d0 0x55c647fa6f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647fa0ad0 0x55c647fa5500 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647fa5b80 con 0x55c647f952d0 2023-07-14T11:15:47.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647fa5cf0 con 0x55c647f95be0 2023-07-14T11:15:47.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c647fa5eb0 con 0x55c647fa0ad0 2023-07-14T11:15:47.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd6980cfb80 10 client.0 _ll_register_callbacks cb 0x7fd66c00c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:15:47.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48326/0 (socket says 172.21.15.40:48326) 2023-07-14T11:15:47.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 -- 172.21.15.40:0/4194239138 learned_addr learned my addr 172.21.15.40:0/4194239138 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:15:47.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd68259b700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f952d0 0x55c647fa6f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd682d9c700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647fa0ad0 0x55c647fa5500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 -- 172.21.15.40:0/4194239138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647fa0ad0 msgr2=0x55c647fa5500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647fa0ad0 0x55c647fa5500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 -- 172.21.15.40:0/4194239138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f952d0 msgr2=0x55c647fa6f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:15:47.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f952d0 0x55c647fa6f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.725+0000 7fd681d9a700 1 -- 172.21.15.40:0/4194239138 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6780114a0 con 0x55c647f95be0 2023-07-14T11:15:47.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd681d9a700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fd66c099950 tx=0x7fd66c0960c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4f7a49328ccf5838 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:15:47.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd665ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:15:47.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd66c0b2070 con 0x55c647f95be0 2023-07-14T11:15:47.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7fd66c0ca550 con 0x55c647f95be0 2023-07-14T11:15:47.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fd678011190 con 0x55c647f95be0 2023-07-14T11:15:47.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=37) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd66c0ca710 con 0x55c647f95be0 2023-07-14T11:15:47.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd6980cfb80 10 client.26276 fetch_fsmap learned FSMap version 37 2023-07-14T11:15:47.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=37,monmap=4+}) v3 -- 0x55c647d3c250 con 0x55c647f95be0 2023-07-14T11:15:47.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.726+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fd678011190 con 0x55c647f95be0 2023-07-14T11:15:47.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 37) v1 ==== 7502+0+0 (secure 0 0 0) 0x7fd66c099140 con 0x55c647f95be0 2023-07-14T11:15:47.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 10 client.26276 fetch_fsmap finished waiting for FSMap version 37 2023-07-14T11:15:47.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 10 client.26276 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:15:47.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 20 client.26276 populate_metadata read hostname 'smithi040' 2023-07-14T11:15:47.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c6480053a0 0x55c6480077b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:15:47.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 4 client.26276 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:15:47.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c647ea3050 con 0x55c6480053a0 2023-07-14T11:15:47.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd68259b700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c6480053a0 0x55c6480077b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:15:47.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd68259b700 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c6480053a0 0x55c6480077b0 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3aaa74072ea62025 server_cookie=563dbede0686253b in_seq=0 out_seq=0 2023-07-14T11:15:47.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.727+0000 7fd665ffb700 10 client.26276 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:47.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd665ffb700 1 -- 172.21.15.40:0/4194239138 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55c647ea3050 con 0x55c6480053a0 2023-07-14T11:15:47.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd665ffb700 10 client.26276 handle_command_reply: tid=0 2023-07-14T11:15:47.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 client.26276 shutdown 2023-07-14T11:15:47.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 msgr2=0x55c647fa7490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7fd66c099950 tx=0x7fd66c0960c0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 shutdown_connections 2023-07-14T11:15:47.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c6480053a0 0x55c6480077b0 crc :-1 s=READY pgs=86 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c647fa0ad0 0x55c647fa5500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c647f952d0 0x55c647fa6f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd665ffb700 0 client.26276 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:15:47.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 --2- 172.21.15.40:0/4194239138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c647f95be0 0x55c647fa7490 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 >> 172.21.15.40:0/4194239138 conn(0x55c647f8bda0 msgr2=0x55c647f908b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 shutdown_connections 2023-07-14T11:15:47.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 1 -- 172.21.15.40:0/4194239138 wait complete. 2023-07-14T11:15:47.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.728+0000 7fd6980cfb80 20 client.26276 trim_cache size 0 max 16384 2023-07-14T11:15:47.744 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:15:47.744 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:15:47.745 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:15:47.745 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:15:47.745 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:15:47.745 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:15:47.745 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:15:47.746 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:15:47.746 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:15:47.746 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:15:47.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 msgr2=0x7fd6680a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 0x7fd6680a3770 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fd674009fd0 tx=0x7fd6740094c0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 msgr2=0x7fd67c198430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:15:47.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fd66c00e9f0 tx=0x7fd66c00ed00 comp rx=0 tx=0).stop 2023-07-14T11:15:47.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 shutdown_connections 2023-07-14T11:15:47.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fd6680a12c0 0x7fd6680a3770 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c10a040 0x7fd67c19c890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fd67c100d00 0x7fd67c197ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 --2- 172.21.15.40:0/2146790540 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fd67c101610 0x7fd67c198430 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:15:47.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 >> 172.21.15.40:0/2146790540 conn(0x7fd67c0fa4b0 msgr2=0x7fd67c0fa880 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:15:47.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 shutdown_connections 2023-07-14T11:15:47.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:15:47.729+0000 7fd6847ff700 1 -- 172.21.15.40:0/2146790540 wait complete. 2023-07-14T11:15:47.756 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:47 smithi040 ceph-mon[97954]: pgmap v112: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:47.759 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:15:48.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:47 smithi130 ceph-mon[129405]: pgmap v112: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:47 smithi045 ceph-mon[128791]: pgmap v112: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:49 smithi130 ceph-mon[129405]: pgmap v113: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:49 smithi040 ceph-mon[97954]: pgmap v113: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:49 smithi045 ceph-mon[128791]: pgmap v113: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 11 KiB/s wr, 12 op/s 2023-07-14T11:15:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:51 smithi130 ceph-mon[129405]: pgmap v114: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 26 KiB/s wr, 22 op/s 2023-07-14T11:15:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:51 smithi040 ceph-mon[97954]: pgmap v114: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 26 KiB/s wr, 22 op/s 2023-07-14T11:15:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:51 smithi045 ceph-mon[128791]: pgmap v114: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 28 KiB/s rd, 26 KiB/s wr, 22 op/s 2023-07-14T11:15:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:53 smithi130 ceph-mon[129405]: pgmap v115: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 KiB/s wr, 17 op/s 2023-07-14T11:15:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:53 smithi040 ceph-mon[97954]: pgmap v115: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 KiB/s wr, 17 op/s 2023-07-14T11:15:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:53 smithi045 ceph-mon[128791]: pgmap v115: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 22 KiB/s wr, 17 op/s 2023-07-14T11:15:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:55 smithi130 ceph-mon[129405]: pgmap v116: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 300 KiB/s wr, 100 op/s 2023-07-14T11:15:55.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:55 smithi040 ceph-mon[97954]: pgmap v116: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 300 KiB/s wr, 100 op/s 2023-07-14T11:15:55.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:55 smithi045 ceph-mon[128791]: pgmap v116: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 246 KiB/s rd, 300 KiB/s wr, 100 op/s 2023-07-14T11:15:56.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:56 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:56.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:56 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:56.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:56 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:15:57.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:57 smithi130 ceph-mon[129405]: pgmap v117: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:15:57.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:57 smithi040 ceph-mon[97954]: pgmap v117: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:15:57.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:57 smithi045 ceph-mon[128791]: pgmap v117: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:16:00.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:15:59 smithi130 ceph-mon[129405]: pgmap v118: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:16:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:15:59 smithi040 ceph-mon[97954]: pgmap v118: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:16:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:15:59 smithi045 ceph-mon[128791]: pgmap v118: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 239 KiB/s rd, 293 KiB/s wr, 93 op/s 2023-07-14T11:16:01.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:01.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:01 smithi130 ceph-mon[129405]: pgmap v119: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 249 KiB/s rd, 305 KiB/s wr, 101 op/s 2023-07-14T11:16:01.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:01.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:01 smithi040 ceph-mon[97954]: pgmap v119: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 249 KiB/s rd, 305 KiB/s wr, 101 op/s 2023-07-14T11:16:01.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:01 smithi045 ceph-mon[128791]: pgmap v119: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 249 KiB/s rd, 305 KiB/s wr, 101 op/s 2023-07-14T11:16:04.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:03 smithi130 ceph-mon[129405]: pgmap v120: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 233 KiB/s rd, 290 KiB/s wr, 90 op/s 2023-07-14T11:16:04.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:03 smithi040 ceph-mon[97954]: pgmap v120: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 233 KiB/s rd, 290 KiB/s wr, 90 op/s 2023-07-14T11:16:04.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:03 smithi045 ceph-mon[128791]: pgmap v120: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 233 KiB/s rd, 290 KiB/s wr, 90 op/s 2023-07-14T11:16:05.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:05 smithi130 ceph-mon[129405]: pgmap v121: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 379 KiB/s rd, 450 KiB/s wr, 113 op/s 2023-07-14T11:16:05.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:05 smithi040 ceph-mon[97954]: pgmap v121: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 379 KiB/s rd, 450 KiB/s wr, 113 op/s 2023-07-14T11:16:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:05 smithi045 ceph-mon[128791]: pgmap v121: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 379 KiB/s rd, 450 KiB/s wr, 113 op/s 2023-07-14T11:16:06.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:06 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:06.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:06 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:06.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:06 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:07.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:07 smithi130 ceph-mon[129405]: pgmap v122: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:07.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:07 smithi040 ceph-mon[97954]: pgmap v122: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:07.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:07 smithi045 ceph-mon[128791]: pgmap v122: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:09 smithi130 ceph-mon[129405]: pgmap v123: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:09 smithi040 ceph-mon[97954]: pgmap v123: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:09 smithi045 ceph-mon[128791]: pgmap v123: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 156 KiB/s rd, 172 KiB/s wr, 29 op/s 2023-07-14T11:16:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:11 smithi130 ceph-mon[129405]: pgmap v124: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 319 KiB/s rd, 395 KiB/s wr, 50 op/s 2023-07-14T11:16:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:11 smithi040 ceph-mon[97954]: pgmap v124: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 319 KiB/s rd, 395 KiB/s wr, 50 op/s 2023-07-14T11:16:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:11 smithi045 ceph-mon[128791]: pgmap v124: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 319 KiB/s rd, 395 KiB/s wr, 50 op/s 2023-07-14T11:16:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:13 smithi130 ceph-mon[129405]: pgmap v125: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 309 KiB/s rd, 384 KiB/s wr, 42 op/s 2023-07-14T11:16:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:13 smithi040 ceph-mon[97954]: pgmap v125: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 309 KiB/s rd, 384 KiB/s wr, 42 op/s 2023-07-14T11:16:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:13 smithi045 ceph-mon[128791]: pgmap v125: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 309 KiB/s rd, 384 KiB/s wr, 42 op/s 2023-07-14T11:16:15.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:14 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:15.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:14 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:14 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:16.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:15 smithi130 ceph-mon[129405]: pgmap v126: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 538 KiB/s wr, 65 op/s 2023-07-14T11:16:16.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:15 smithi040 ceph-mon[97954]: pgmap v126: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 538 KiB/s wr, 65 op/s 2023-07-14T11:16:16.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:15 smithi045 ceph-mon[128791]: pgmap v126: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 1.1 MiB/s rd, 538 KiB/s wr, 65 op/s 2023-07-14T11:16:17.761 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:16:17.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 -- 172.21.15.40:0/1426658819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280ff1c0 msgr2=0x7f532810be90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:17.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 --2- 172.21.15.40:0/1426658819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280ff1c0 0x7f532810be90 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7f5318009990 tx=0x7f5318009ca0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 -- 172.21.15.40:0/1426658819 shutdown_connections 2023-07-14T11:16:17.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 --2- 172.21.15.40:0/1426658819 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f532810c3d0 0x7f532810e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 --2- 172.21.15.40:0/1426658819 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f53280fe820 0x7f53280febf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 --2- 172.21.15.40:0/1426658819 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280ff1c0 0x7f532810be90 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 -- 172.21.15.40:0/1426658819 >> 172.21.15.40:0/1426658819 conn(0x7f53280fa4b0 msgr2=0x7f53280fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:17.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 -- 172.21.15.40:0/1426658819 shutdown_connections 2023-07-14T11:16:17.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.980+0000 7f532f9ff700 1 -- 172.21.15.40:0/1426658819 wait complete. 2023-07-14T11:16:17.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.981+0000 7f532f9ff700 1 Processor -- start 2023-07-14T11:16:17.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.981+0000 7f532f9ff700 1 -- start start 2023-07-14T11:16:17.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 0x7f5328198320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 0x7f532819c780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f532819cdc0 con 0x7f532810c3d0 2023-07-14T11:16:17.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f532819cf30 con 0x7f53280ff1c0 2023-07-14T11:16:17.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532f9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f532819d0f0 con 0x7f53280fe820 2023-07-14T11:16:17.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532d79b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532d79b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:41374/0 (socket says 172.21.15.40:41374) 2023-07-14T11:16:17.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532d79b700 1 -- 172.21.15.40:0/603919754 learned_addr learned my addr 172.21.15.40:0/603919754 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:17.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.982+0000 7f532df9c700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 0x7f532819c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532cf9a700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 0x7f5328198320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 -- 172.21.15.40:0/603919754 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 msgr2=0x7f5328198320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:17.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 0x7f5328198320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 -- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 msgr2=0x7f532819c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:17.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 0x7f532819c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 -- 172.21.15.40:0/603919754 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f53180095f0 con 0x7f53280fe820 2023-07-14T11:16:17.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532df9c700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 0x7f532819c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:16:17.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532cf9a700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 0x7f5328198320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:16:17.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532d79b700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f532400d980 tx=0x7f532400dd40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:17.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f531e7fc700 1 -- 172.21.15.40:0/603919754 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f532400bb60 con 0x7f53280fe820 2023-07-14T11:16:17.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f531e7fc700 1 -- 172.21.15.40:0/603919754 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5324007640 con 0x7f53280fe820 2023-07-14T11:16:17.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f532819d3d0 con 0x7f53280fe820 2023-07-14T11:16:17.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f531e7fc700 1 -- 172.21.15.40:0/603919754 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5324004e90 con 0x7f53280fe820 2023-07-14T11:16:17.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.983+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53281a4c10 con 0x7f53280fe820 2023-07-14T11:16:17.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.989+0000 7f531e7fc700 1 -- 172.21.15.40:0/603919754 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f532400bcc0 con 0x7f53280fe820 2023-07-14T11:16:17.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.990+0000 7f531e7fc700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 0x7f53140a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.990+0000 7f531e7fc700 1 -- 172.21.15.40:0/603919754 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f53240cce80 con 0x7f53280fe820 2023-07-14T11:16:17.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.990+0000 7f5343323b80 1 Processor -- start 2023-07-14T11:16:17.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.991+0000 7f532cf9a700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 0x7f53140a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.991+0000 7f5343323b80 1 -- start start 2023-07-14T11:16:17.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.991+0000 7f532cf9a700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 0x7f53140a37f0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f531800b2e0 tx=0x7f5318005e80 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:17.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 0x562990438590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562990438ad0 0x56299043aeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:17.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990333cb0 con 0x562990438ad0 2023-07-14T11:16:17.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990386a40 con 0x56299042d2d0 2023-07-14T11:16:17.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f5343323b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299037fed0 con 0x56299042dbe0 2023-07-14T11:16:17.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50032/0 (socket says 172.21.15.40:50032) 2023-07-14T11:16:17.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 -- 172.21.15.40:0/1659502521 learned_addr learned my addr 172.21.15.40:0/1659502521 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:17.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532cf9a700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 0x562990438590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532df9c700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562990438ad0 0x56299043aeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:17.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 -- 172.21.15.40:0/1659502521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 msgr2=0x562990438590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:17.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 0x562990438590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 -- 172.21.15.40:0/1659502521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562990438ad0 msgr2=0x56299043aeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:17.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562990438ad0 0x56299043aeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:17.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532d79b700 1 -- 172.21.15.40:0/1659502521 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5629901adde0 con 0x56299042d2d0 2023-07-14T11:16:17.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.992+0000 7f532cf9a700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 0x562990438590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:16:17.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.993+0000 7f532d79b700 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f53240a1b40 tx=0x7f532400df30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d65a6432b76a7c1b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:17.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.993+0000 7f53137fe700 1 -- 172.21.15.40:0/1659502521 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f53240a2070 con 0x56299042d2d0 2023-07-14T11:16:18.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.993+0000 7f53137fe700 1 -- 172.21.15.40:0/1659502521 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f53240b2210 con 0x56299042d2d0 2023-07-14T11:16:18.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.993+0000 7f53137fe700 1 -- 172.21.15.40:0/1659502521 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5324099370 con 0x56299042d2d0 2023-07-14T11:16:18.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 -- 172.21.15.40:0/1659502521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 msgr2=0x56299042d6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f53240a1b40 tx=0x7f532400df30 comp rx=0 tx=0).stop 2023-07-14T11:16:18.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 -- 172.21.15.40:0/1659502521 shutdown_connections 2023-07-14T11:16:18.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562990438ad0 0x56299043aeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x56299042d6a0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 --2- 172.21.15.40:0/1659502521 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042dbe0 0x562990438590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 -- 172.21.15.40:0/1659502521 >> 172.21.15.40:0/1659502521 conn(0x562990423da0 msgr2=0x5629904296a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:18.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 -- 172.21.15.40:0/1659502521 shutdown_connections 2023-07-14T11:16:18.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.994+0000 7f5343323b80 1 -- 172.21.15.40:0/1659502521 wait complete. 2023-07-14T11:16:18.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.995+0000 7f5343323b80 1 -- start start 2023-07-14T11:16:18.004 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.995+0000 7f5343323b80 1 Processor -- start 2023-07-14T11:16:18.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 0x562990435820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x562990435d60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299031fac0 con 0x56299042dbe0 2023-07-14T11:16:18.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299031f7c0 con 0x562990438ad0 2023-07-14T11:16:18.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990203920 con 0x56299042d2d0 2023-07-14T11:16:18.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f5343323b80 10 client.0 _ll_register_callbacks cb 0x56299031d760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:16:18.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f532df9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f532df9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50044/0 (socket says 172.21.15.40:50044) 2023-07-14T11:16:18.006 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.996+0000 7f532df9c700 1 -- 172.21.15.40:0/2714479387 learned_addr learned my addr 172.21.15.40:0/2714479387 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:18.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532d79b700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 0x562990435820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532cf9a700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x562990435d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 -- 172.21.15.40:0/2714479387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 msgr2=0x562990435820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 0x562990435820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 -- 172.21.15.40:0/2714479387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 msgr2=0x562990435d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x562990435d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 -- 172.21.15.40:0/2714479387 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f53281a4c10 con 0x562990438ad0 2023-07-14T11:16:18.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532d79b700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 0x562990435820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:16:18.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f532df9c700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f5320007a40 tx=0x7f5320007d50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=31cd9abc9b9de65f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:18.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.997+0000 7f5310ff9700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:16:18.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f532000e070 con 0x562990438ad0 2023-07-14T11:16:18.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5629902d94f0 con 0x562990438ad0 2023-07-14T11:16:18.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5320003c00 con 0x562990438ad0 2023-07-14T11:16:18.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=42) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5320009e50 con 0x562990438ad0 2023-07-14T11:16:18.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5343323b80 10 client.26285 fetch_fsmap learned FSMap version 42 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=42,monmap=4+}) v3 -- 0x562990196c60 con 0x562990438ad0 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.998+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5629902d94f0 con 0x562990438ad0 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 42) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f532001c020 con 0x562990438ad0 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 10 client.26285 fetch_fsmap finished waiting for FSMap version 42 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 10 client.26285 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:16:18.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 20 client.26285 populate_metadata read hostname 'smithi040' 2023-07-14T11:16:18.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629904484a0 0x56299044a8b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 4 client.26285 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:16:18.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562990378eb0 con 0x5629904484a0 2023-07-14T11:16:18.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f532d79b700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629904484a0 0x56299044a8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f532d79b700 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629904484a0 0x56299044a8b0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=62d1b156de5e341c server_cookie=9df5bc426861e6f in_seq=0 out_seq=0 2023-07-14T11:16:18.012 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:17.999+0000 7f5310ff9700 10 client.26285 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:18.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.000+0000 7f5310ff9700 1 -- 172.21.15.40:0/2714479387 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562990378eb0 con 0x5629904484a0 2023-07-14T11:16:18.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.000+0000 7f5310ff9700 10 client.26285 handle_command_reply: tid=0 2023-07-14T11:16:18.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.000+0000 7f5343323b80 1 client.26285 shutdown 2023-07-14T11:16:18.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 msgr2=0x5629904362a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.013 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f5320007a40 tx=0x7f5320007d50 comp rx=0 tx=0).stop 2023-07-14T11:16:18.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 shutdown_connections 2023-07-14T11:16:18.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629904484a0 0x56299044a8b0 crc :-1 s=READY pgs=87 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562990438ad0 0x5629904362a0 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56299042d2d0 0x562990435820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 --2- 172.21.15.40:0/2714479387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x562990435d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 >> 172.21.15.40:0/2714479387 conn(0x562990423da0 msgr2=0x562990429230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:18.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5310ff9700 0 client.26285 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:18.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 shutdown_connections 2023-07-14T11:16:18.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 1 -- 172.21.15.40:0/2714479387 wait complete. 2023-07-14T11:16:18.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.001+0000 7f5343323b80 20 client.26285 trim_cache size 0 max 16384 2023-07-14T11:16:18.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 Processor -- start 2023-07-14T11:16:18.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 -- start start 2023-07-14T11:16:18.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042d2d0 0x5629904301b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 0x56299043f4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x562990430b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990425ec0 con 0x56299042d2d0 2023-07-14T11:16:18.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299042e160 con 0x56299042dbe0 2023-07-14T11:16:18.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.019+0000 7f5343323b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990444e10 con 0x562990438ad0 2023-07-14T11:16:18.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532cf9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 0x56299043f4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532d79b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042d2d0 0x5629904301b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532cf9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 0x56299043f4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50058/0 (socket says 172.21.15.40:50058) 2023-07-14T11:16:18.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x562990430b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532cf9a700 1 -- 172.21.15.40:0/584715466 learned_addr learned my addr 172.21.15.40:0/584715466 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:18.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 -- 172.21.15.40:0/584715466 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 msgr2=0x56299043f4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 0x56299043f4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 -- 172.21.15.40:0/584715466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042d2d0 msgr2=0x5629904301b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042d2d0 0x5629904301b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 -- 172.21.15.40:0/584715466 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5629904255e0 con 0x562990438ad0 2023-07-14T11:16:18.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.020+0000 7f532df9c700 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x562990430b30 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f532000b010 tx=0x7f5320009ba0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=63bee7a585ba54e6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:18.028 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5313fff700 1 -- 172.21.15.40:0/584715466 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f532000e040 con 0x562990438ad0 2023-07-14T11:16:18.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5313fff700 1 -- 172.21.15.40:0/584715466 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f532000f040 con 0x562990438ad0 2023-07-14T11:16:18.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5313fff700 1 -- 172.21.15.40:0/584715466 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f532001f7d0 con 0x562990438ad0 2023-07-14T11:16:18.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 -- 172.21.15.40:0/584715466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 msgr2=0x562990430b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x562990430b30 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f532000b010 tx=0x7f5320009ba0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 -- 172.21.15.40:0/584715466 shutdown_connections 2023-07-14T11:16:18.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x562990430b30 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042d2d0 0x5629904301b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 --2- 172.21.15.40:0/584715466 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042dbe0 0x56299043f4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.021+0000 7f5343323b80 1 -- 172.21.15.40:0/584715466 >> 172.21.15.40:0/584715466 conn(0x562990423da0 msgr2=0x562990427e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:18.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.022+0000 7f5343323b80 1 -- 172.21.15.40:0/584715466 shutdown_connections 2023-07-14T11:16:18.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.022+0000 7f5343323b80 1 -- 172.21.15.40:0/584715466 wait complete. 2023-07-14T11:16:18.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.022+0000 7f5343323b80 1 -- start start 2023-07-14T11:16:18.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.022+0000 7f5343323b80 1 Processor -- start 2023-07-14T11:16:18.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x5629904909e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x56299049b330 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562990491110 con 0x56299042dbe0 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299049b870 con 0x56299042d2d0 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56299049ba00 con 0x562990438ad0 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f5343323b80 10 client.0 _ll_register_callbacks cb 0x7f5320009450 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:16:18.032 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50062/0 (socket says 172.21.15.40:50062) 2023-07-14T11:16:18.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 -- 172.21.15.40:0/1220067821 learned_addr learned my addr 172.21.15.40:0/1220067821 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:18.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532df9c700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x56299049b330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532cf9a700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x5629904909e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.033 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 -- 172.21.15.40:0/1220067821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 msgr2=0x56299049b330 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x56299049b330 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 -- 172.21.15.40:0/1220067821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 msgr2=0x5629904909e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:18.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x5629904909e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.023+0000 7f532d79b700 1 -- 172.21.15.40:0/1220067821 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562990196c60 con 0x56299042d2d0 2023-07-14T11:16:18.034 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f532d79b700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5324096690 tx=0x7f5324099b20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5a59dc9219da1d20 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:18.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5310ff9700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:16:18.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5324099310 con 0x56299042d2d0 2023-07-14T11:16:18.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f53240a2070 con 0x56299042d2d0 2023-07-14T11:16:18.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f53240b2210 con 0x56299042d2d0 2023-07-14T11:16:18.035 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=42) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5324098040 con 0x56299042d2d0 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5343323b80 10 client.26291 fetch_fsmap learned FSMap version 42 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=42,monmap=4+}) v3 -- 0x5629901d4250 con 0x56299042d2d0 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.024+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f53240bd080 con 0x56299042d2d0 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 42) v1 ==== 7486+0+0 (secure 0 0 0) 0x7f5324098b80 con 0x56299042d2d0 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 10 client.26291 fetch_fsmap finished waiting for FSMap version 42 2023-07-14T11:16:18.036 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 10 client.26291 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:16:18.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 20 client.26291 populate_metadata read hostname 'smithi040' 2023-07-14T11:16:18.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56299049cdd0 0x56299049f1e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:18.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 4 client.26291 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:16:18.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56299033b050 con 0x56299049cdd0 2023-07-14T11:16:18.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.025+0000 7f532cf9a700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56299049cdd0 0x56299049f1e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:18.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5310ff9700 10 client.26291 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:18.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f532cf9a700 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56299049cdd0 0x56299049f1e0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=fd251293a53cf095 server_cookie=80e9ff098127e506 in_seq=0 out_seq=0 2023-07-14T11:16:18.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5310ff9700 1 -- 172.21.15.40:0/1220067821 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x56299033b050 con 0x56299049cdd0 2023-07-14T11:16:18.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5310ff9700 10 client.26291 handle_command_reply: tid=0 2023-07-14T11:16:18.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5343323b80 1 client.26291 shutdown 2023-07-14T11:16:18.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 msgr2=0x5629904904a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.026+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f5324096690 tx=0x7f5324099b20 comp rx=0 tx=0).stop 2023-07-14T11:16:18.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 shutdown_connections 2023-07-14T11:16:18.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56299049cdd0 0x56299049f1e0 crc :-1 s=READY pgs=88 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562990438ad0 0x56299049b330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.040 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56299042d2d0 0x5629904904a0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 --2- 172.21.15.40:0/1220067821 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56299042dbe0 0x5629904909e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 >> 172.21.15.40:0/1220067821 conn(0x562990423da0 msgr2=0x562990430b30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:18.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5310ff9700 0 client.26291 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:18.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 shutdown_connections 2023-07-14T11:16:18.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 1 -- 172.21.15.40:0/1220067821 wait complete. 2023-07-14T11:16:18.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f5343323b80 20 client.26291 trim_cache size 0 max 16384 2023-07-14T11:16:18.042 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:16:18.042 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:16:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:16:18.044 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:16:18.044 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:16:18.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 msgr2=0x7f53140a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 0x7f53140a37f0 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f531800b2e0 tx=0x7f5318005e80 comp rx=0 tx=0).stop 2023-07-14T11:16:18.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 msgr2=0x7f5328197de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:18.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.027+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f532400d980 tx=0x7f532400dd40 comp rx=0 tx=0).stop 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 shutdown_connections 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f53140a1340 0x7f53140a37f0 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f532810c3d0 0x7f532819c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f53280fe820 0x7f5328197de0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 --2- 172.21.15.40:0/603919754 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f53280ff1c0 0x7f5328198320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:18.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 >> 172.21.15.40:0/603919754 conn(0x7f53280fa4b0 msgr2=0x7f532810d690 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:18.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 shutdown_connections 2023-07-14T11:16:18.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:18.028+0000 7f532f9ff700 1 -- 172.21.15.40:0/603919754 wait complete. 2023-07-14T11:16:18.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:17 smithi130 ceph-mon[129405]: pgmap v127: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:18.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:17 smithi040 ceph-mon[97954]: pgmap v127: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:18.049 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:16:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:17 smithi045 ceph-mon[128791]: pgmap v127: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:20.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:19 smithi130 ceph-mon[129405]: pgmap v128: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:20.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:19 smithi040 ceph-mon[97954]: pgmap v128: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:20.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:19 smithi045 ceph-mon[128791]: pgmap v128: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 938 KiB/s rd, 378 KiB/s wr, 43 op/s 2023-07-14T11:16:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:21 smithi130 ceph-mon[129405]: pgmap v129: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 985 KiB/s rd, 398 KiB/s wr, 51 op/s 2023-07-14T11:16:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:21 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:16:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:21 smithi040 ceph-mon[97954]: pgmap v129: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 985 KiB/s rd, 398 KiB/s wr, 51 op/s 2023-07-14T11:16:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:21 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:16:22.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:21 smithi045 ceph-mon[128791]: pgmap v129: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 985 KiB/s rd, 398 KiB/s wr, 51 op/s 2023-07-14T11:16:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:21 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:16:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:22 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:16:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:22 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:16:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:22 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:16:23.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:22 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:16:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:22 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:16:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:22 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:16:23.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:22 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:16:23.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:22 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:16:23.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:22 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:16:24.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:23 smithi130 ceph-mon[129405]: pgmap v130: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 822 KiB/s rd, 174 KiB/s wr, 31 op/s 2023-07-14T11:16:24.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:23 smithi040 ceph-mon[97954]: pgmap v130: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 822 KiB/s rd, 174 KiB/s wr, 31 op/s 2023-07-14T11:16:24.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:23 smithi045 ceph-mon[128791]: pgmap v130: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 822 KiB/s rd, 174 KiB/s wr, 31 op/s 2023-07-14T11:16:25.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:25 smithi130 ceph-mon[129405]: pgmap v131: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 833 KiB/s rd, 181 KiB/s wr, 38 op/s 2023-07-14T11:16:25.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:25 smithi040 ceph-mon[97954]: pgmap v131: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 833 KiB/s rd, 181 KiB/s wr, 38 op/s 2023-07-14T11:16:25.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:25 smithi045 ceph-mon[128791]: pgmap v131: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 833 KiB/s rd, 181 KiB/s wr, 38 op/s 2023-07-14T11:16:28.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:27 smithi130 ceph-mon[129405]: pgmap v132: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:28.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:27 smithi040 ceph-mon[97954]: pgmap v132: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:28.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:27 smithi045 ceph-mon[128791]: pgmap v132: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:30.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:29 smithi130 ceph-mon[129405]: pgmap v133: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:29 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:30.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:29 smithi040 ceph-mon[97954]: pgmap v133: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:29 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:29 smithi045 ceph-mon[128791]: pgmap v133: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 58 KiB/s rd, 27 KiB/s wr, 15 op/s 2023-07-14T11:16:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:29 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:31.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:30 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:30 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:31.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:30 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:30 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:31.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:30 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:30 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:31 smithi130 ceph-mon[129405]: pgmap v134: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 35 KiB/s wr, 166 op/s 2023-07-14T11:16:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:31 smithi040 ceph-mon[97954]: pgmap v134: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 35 KiB/s wr, 166 op/s 2023-07-14T11:16:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:32.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:31 smithi045 ceph-mon[128791]: pgmap v134: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 35 KiB/s wr, 166 op/s 2023-07-14T11:16:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:34.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:33 smithi130 ceph-mon[129405]: pgmap v135: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 16 KiB/s wr, 158 op/s 2023-07-14T11:16:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:33 smithi040 ceph-mon[97954]: pgmap v135: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 16 KiB/s wr, 158 op/s 2023-07-14T11:16:34.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:33 smithi045 ceph-mon[128791]: pgmap v135: 129 pgs: 129 active+clean; 6.5 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 13 MiB/s rd, 16 KiB/s wr, 158 op/s 2023-07-14T11:16:35.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:35 smithi130 ceph-mon[129405]: pgmap v136: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 448 op/s 2023-07-14T11:16:35.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:35 smithi040 ceph-mon[97954]: pgmap v136: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 448 op/s 2023-07-14T11:16:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:35 smithi045 ceph-mon[128791]: pgmap v136: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 448 op/s 2023-07-14T11:16:36.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:36 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:36 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:36.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:36 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:36.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:36 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:36.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:36 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:36.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:36 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:36.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:36 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:36.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:36 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:36.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:36 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:37.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:37 smithi130 ceph-mon[129405]: pgmap v137: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:37.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:37 smithi040 ceph-mon[97954]: pgmap v137: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:37.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:37 smithi045 ceph-mon[128791]: pgmap v137: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:38.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:38 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:38 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:38 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:38.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:38 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:38 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:38 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:38.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:38 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:38 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:38 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:39 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:39 smithi130 ceph-mon[129405]: pgmap v138: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:39.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:39 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:39.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:39 smithi040 ceph-mon[97954]: pgmap v138: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:39.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:39 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:39 smithi045 ceph-mon[128791]: pgmap v138: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 15 MiB/s rd, 5.3 MiB/s wr, 441 op/s 2023-07-14T11:16:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:41 smithi130 ceph-mon[129405]: pgmap v139: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 MiB/s rd, 5.3 MiB/s wr, 450 op/s 2023-07-14T11:16:42.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:41 smithi040 ceph-mon[97954]: pgmap v139: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 MiB/s rd, 5.3 MiB/s wr, 450 op/s 2023-07-14T11:16:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:41 smithi045 ceph-mon[128791]: pgmap v139: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 17 MiB/s rd, 5.3 MiB/s wr, 450 op/s 2023-07-14T11:16:44.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:43 smithi130 ceph-mon[129405]: pgmap v140: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.3 MiB/s wr, 299 op/s 2023-07-14T11:16:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:44.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:43 smithi040 ceph-mon[97954]: pgmap v140: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.3 MiB/s wr, 299 op/s 2023-07-14T11:16:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:44.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:43 smithi045 ceph-mon[128791]: pgmap v140: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 3.9 MiB/s rd, 5.3 MiB/s wr, 299 op/s 2023-07-14T11:16:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:16:45.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:45 smithi130 ceph-mon[129405]: pgmap v141: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 8.3 MiB/s wr, 562 op/s 2023-07-14T11:16:45.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:45 smithi040 ceph-mon[97954]: pgmap v141: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 8.3 MiB/s wr, 562 op/s 2023-07-14T11:16:45.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:45 smithi045 ceph-mon[128791]: pgmap v141: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 8.3 MiB/s wr, 562 op/s 2023-07-14T11:16:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:46 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:46.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:46 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:46.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:46 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:46.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:46 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:46.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:46 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:46 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:47 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:47 smithi130 ceph-mon[129405]: pgmap v142: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:47.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:47 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:47.804 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:47 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:47.804 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:47 smithi040 ceph-mon[97954]: pgmap v142: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:47.804 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:47 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:47.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:47 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:47 smithi045 ceph-mon[128791]: pgmap v142: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:47 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:48.050 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:16:48.296 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/892983729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 msgr2=0x7f1d2810be70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/892983729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2810be70 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f1d18009990 tx=0x7f1d18009ca0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/892983729 shutdown_connections 2023-07-14T11:16:48.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/892983729 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d2810c3b0 0x7f1d2810e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.297 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/892983729 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d280fe800 0x7f1d280febd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/892983729 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2810be70 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/892983729 >> 172.21.15.40:0/892983729 conn(0x7f1d280745d0 msgr2=0x7f1d280749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/892983729 shutdown_connections 2023-07-14T11:16:48.298 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.294+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/892983729 wait complete. 2023-07-14T11:16:48.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.295+0000 7f1d2eb9d700 1 Processor -- start 2023-07-14T11:16:48.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.295+0000 7f1d2eb9d700 1 -- start start 2023-07-14T11:16:48.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2819df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d2810c3b0 0x7f1d28197ac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d28198130 con 0x7f1d2810c3b0 2023-07-14T11:16:48.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d281982a0 con 0x7f1d280fe800 2023-07-14T11:16:48.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2eb9d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d28198460 con 0x7f1d280ff110 2023-07-14T11:16:48.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58376/0 (socket says 172.21.15.40:58376) 2023-07-14T11:16:48.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3768311089 learned_addr learned my addr 172.21.15.40:0/3768311089 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:48.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2819df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d2810c3b0 0x7f1d28197ac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3768311089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 msgr2=0x7f1d2819df90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2819df90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d2810c3b0 msgr2=0x7f1d28197ac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d2810c3b0 0x7f1d28197ac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3768311089 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d24009520 con 0x7f1d280fe800 2023-07-14T11:16:48.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.296+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2819df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:16:48.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f1d2400eb30 tx=0x7f1d2400ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d1effd700 1 -- 172.21.15.40:0/3768311089 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d24009880 con 0x7f1d280fe800 2023-07-14T11:16:48.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d180095f0 con 0x7f1d280fe800 2023-07-14T11:16:48.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d1effd700 1 -- 172.21.15.40:0/3768311089 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d240044e0 con 0x7f1d280fe800 2023-07-14T11:16:48.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d1effd700 1 -- 172.21.15.40:0/3768311089 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d24003e90 con 0x7f1d280fe800 2023-07-14T11:16:48.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.297+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d28198930 con 0x7f1d280fe800 2023-07-14T11:16:48.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.301+0000 7f1d1effd700 1 -- 172.21.15.40:0/3768311089 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f1d24004000 con 0x7f1d280fe800 2023-07-14T11:16:48.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.303+0000 7f1d1effd700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 0x7f1d140a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.303+0000 7f1d1effd700 1 -- 172.21.15.40:0/3768311089 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1d24014070 con 0x7f1d280fe800 2023-07-14T11:16:48.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.303+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 0x7f1d140a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.304+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 0x7f1d140a3810 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1d280fff50 tx=0x7f1d18005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.304+0000 7f1d43756b80 1 Processor -- start 2023-07-14T11:16:48.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.304+0000 7f1d43756b80 1 -- start start 2023-07-14T11:16:48.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620077d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x5562007877f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x55620078a110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556200682cb0 con 0x55620077d4f0 2023-07-14T11:16:48.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562006d5a40 con 0x55620077de00 2023-07-14T11:16:48.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d43756b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562006ceed0 con 0x556200787d30 2023-07-14T11:16:48.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d2d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x5562007877f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d2d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x5562007877f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58384/0 (socket says 172.21.15.40:58384) 2023-07-14T11:16:48.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d2d39a700 1 -- 172.21.15.40:0/1410705056 learned_addr learned my addr 172.21.15.40:0/1410705056 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:48.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x55620078a110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.305+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620077d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 -- 172.21.15.40:0/1410705056 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 msgr2=0x5562007877f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x5562007877f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 -- 172.21.15.40:0/1410705056 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 msgr2=0x55620077d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620077d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 -- 172.21.15.40:0/1410705056 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5562004fcde0 con 0x556200787d30 2023-07-14T11:16:48.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x55620078a110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f1d28073010 tx=0x7f1d20009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9d9d03e8e865c991 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d13fff700 1 -- 172.21.15.40:0/1410705056 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d2000e070 con 0x556200787d30 2023-07-14T11:16:48.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d13fff700 1 -- 172.21.15.40:0/1410705056 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d2000f040 con 0x556200787d30 2023-07-14T11:16:48.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.306+0000 7f1d13fff700 1 -- 172.21.15.40:0/1410705056 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d200147a0 con 0x556200787d30 2023-07-14T11:16:48.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 -- 172.21.15.40:0/1410705056 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 msgr2=0x55620078a110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x55620078a110 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f1d28073010 tx=0x7f1d20009920 comp rx=0 tx=0).stop 2023-07-14T11:16:48.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 -- 172.21.15.40:0/1410705056 shutdown_connections 2023-07-14T11:16:48.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x55620078a110 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620077d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 --2- 172.21.15.40:0/1410705056 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x5562007877f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 -- 172.21.15.40:0/1410705056 >> 172.21.15.40:0/1410705056 conn(0x556200772ff0 msgr2=0x556200778930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 -- 172.21.15.40:0/1410705056 shutdown_connections 2023-07-14T11:16:48.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.307+0000 7f1d43756b80 1 -- 172.21.15.40:0/1410705056 wait complete. 2023-07-14T11:16:48.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.308+0000 7f1d43756b80 1 -- start start 2023-07-14T11:16:48.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.308+0000 7f1d43756b80 1 Processor -- start 2023-07-14T11:16:48.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077de00 0x556200784900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200784ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55620066eac0 con 0x55620077de00 2023-07-14T11:16:48.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55620066e7c0 con 0x55620077d4f0 2023-07-14T11:16:48.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556200552920 con 0x556200787d30 2023-07-14T11:16:48.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d43756b80 10 client.0 _ll_register_callbacks cb 0x55620066c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:16:48.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58386/0 (socket says 172.21.15.40:58386) 2023-07-14T11:16:48.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.309+0000 7f1d2db9b700 1 -- 172.21.15.40:0/908310436 learned_addr learned my addr 172.21.15.40:0/908310436 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:48.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200784ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077de00 0x556200784900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 -- 172.21.15.40:0/908310436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 msgr2=0x556200784ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200784ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 -- 172.21.15.40:0/908310436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077de00 msgr2=0x556200784900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077de00 0x556200784900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 -- 172.21.15.40:0/908310436 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d20009050 con 0x55620077d4f0 2023-07-14T11:16:48.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200784ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:16:48.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f1d24096b40 tx=0x7f1d24099d00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=61a98fd27f9df6c5 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d117fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:16:48.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d2400cc90 con 0x55620077d4f0 2023-07-14T11:16:48.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1d2409a0d0 con 0x55620077d4f0 2023-07-14T11:16:48.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.310+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5562006284f0 con 0x55620077d4f0 2023-07-14T11:16:48.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=46) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d2409e100 con 0x55620077d4f0 2023-07-14T11:16:48.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 10 client.26303 fetch_fsmap learned FSMap version 46 2023-07-14T11:16:48.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=46,monmap=4+}) v3 -- 0x5562004e5c60 con 0x55620077d4f0 2023-07-14T11:16:48.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5562006284f0 con 0x55620077d4f0 2023-07-14T11:16:48.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 46) v1 ==== 7502+0+0 (secure 0 0 0) 0x7f1d24099650 con 0x55620077d4f0 2023-07-14T11:16:48.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 10 client.26303 fetch_fsmap finished waiting for FSMap version 46 2023-07-14T11:16:48.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 10 client.26303 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:16:48.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 20 client.26303 populate_metadata read hostname 'smithi040' 2023-07-14T11:16:48.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x556200797540 0x556200799950 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 4 client.26303 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:16:48.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.311+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5562006c7eb0 con 0x556200797540 2023-07-14T11:16:48.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.312+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x556200797540 0x556200799950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.312+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x556200797540 0x556200799950 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8a234efcffa2a55f server_cookie=8651317a5fcc1095 in_seq=0 out_seq=0 2023-07-14T11:16:48.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.312+0000 7f1d117fa700 10 client.26303 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:48.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d117fa700 1 -- 172.21.15.40:0/908310436 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5562006c7eb0 con 0x556200797540 2023-07-14T11:16:48.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d117fa700 10 client.26303 handle_command_reply: tid=0 2023-07-14T11:16:48.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 client.26303 shutdown 2023-07-14T11:16:48.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 msgr2=0x556200787680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f1d24096b40 tx=0x7f1d24099d00 comp rx=0 tx=0).stop 2023-07-14T11:16:48.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 shutdown_connections 2023-07-14T11:16:48.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x556200797540 0x556200799950 crc :-1 s=READY pgs=89 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200784ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x556200787680 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d117fa700 0 client.26303 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:48.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 --2- 172.21.15.40:0/908310436 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077de00 0x556200784900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 >> 172.21.15.40:0/908310436 conn(0x556200772ff0 msgr2=0x556200778520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 shutdown_connections 2023-07-14T11:16:48.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.313+0000 7f1d43756b80 1 -- 172.21.15.40:0/908310436 wait complete. 2023-07-14T11:16:48.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.314+0000 7f1d43756b80 20 client.26303 trim_cache size 0 max 16384 2023-07-14T11:16:48.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 Processor -- start 2023-07-14T11:16:48.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 -- start start 2023-07-14T11:16:48.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620078e810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200778400 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x556200794d80 con 0x55620077d4f0 2023-07-14T11:16:48.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55620077e980 con 0x55620077de00 2023-07-14T11:16:48.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.331+0000 7f1d43756b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x556200779610 con 0x556200787d30 2023-07-14T11:16:48.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620078e810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58388/0 (socket says 172.21.15.40:58388) 2023-07-14T11:16:48.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 -- 172.21.15.40:0/2294407182 learned_addr learned my addr 172.21.15.40:0/2294407182 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:48.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 -- 172.21.15.40:0/2294407182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 msgr2=0x556200778400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200778400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 -- 172.21.15.40:0/2294407182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 msgr2=0x55620078e810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620078e810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 -- 172.21.15.40:0/2294407182 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x556200523250 con 0x55620077de00 2023-07-14T11:16:48.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.332+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f1d28068750 tx=0x7f1d18020000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=c31e4469e3c51ba0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d1cbff700 1 -- 172.21.15.40:0/2294407182 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d18037070 con 0x55620077de00 2023-07-14T11:16:48.339 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d1cbff700 1 -- 172.21.15.40:0/2294407182 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d1802c040 con 0x55620077de00 2023-07-14T11:16:48.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d1cbff700 1 -- 172.21.15.40:0/2294407182 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d1802b040 con 0x55620077de00 2023-07-14T11:16:48.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 -- 172.21.15.40:0/2294407182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 msgr2=0x556200793980 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f1d28068750 tx=0x7f1d18020000 comp rx=0 tx=0).stop 2023-07-14T11:16:48.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 -- 172.21.15.40:0/2294407182 shutdown_connections 2023-07-14T11:16:48.340 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x556200787d30 0x556200778400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55620077d4f0 0x55620078e810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 --2- 172.21.15.40:0/2294407182 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077de00 0x556200793980 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.333+0000 7f1d43756b80 1 -- 172.21.15.40:0/2294407182 >> 172.21.15.40:0/2294407182 conn(0x556200772ff0 msgr2=0x556200777060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.334+0000 7f1d43756b80 1 -- 172.21.15.40:0/2294407182 shutdown_connections 2023-07-14T11:16:48.341 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.334+0000 7f1d43756b80 1 -- 172.21.15.40:0/2294407182 wait complete. 2023-07-14T11:16:48.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.334+0000 7f1d43756b80 1 -- start start 2023-07-14T11:16:48.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.334+0000 7f1d43756b80 1 Processor -- start 2023-07-14T11:16:48.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55620077de00 0x55620078cc40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.342 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556200787d30 0x55620078d180 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55620078d830 con 0x556200787d30 2023-07-14T11:16:48.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55620078d9a0 con 0x55620077d4f0 2023-07-14T11:16:48.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5562007eb410 con 0x55620077de00 2023-07-14T11:16:48.343 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d43756b80 10 client.0 _ll_register_callbacks cb 0x7f1d240964a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:16:48.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2d39a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55620077de00 0x55620078cc40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58394/0 (socket says 172.21.15.40:58394) 2023-07-14T11:16:48.344 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3134409807 learned_addr learned my addr 172.21.15.40:0/3134409807 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:16:48.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2e39c700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556200787d30 0x55620078d180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3134409807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55620077de00 msgr2=0x55620078cc40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55620077de00 0x55620078cc40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.345 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3134409807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556200787d30 msgr2=0x55620078d180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:16:48.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556200787d30 0x55620078d180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.335+0000 7f1d2db9b700 1 -- 172.21.15.40:0/3134409807 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d2810f2c0 con 0x55620077d4f0 2023-07-14T11:16:48.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d2db9b700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f1d24000c00 tx=0x7f1d24098680 comp rx=0 tx=0).ready entity=mon.2 client_cookie=30d60518a279b7f9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:16:48.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d117fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:16:48.346 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d24099a30 con 0x55620077d4f0 2023-07-14T11:16:48.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f1d2400cc90 con 0x55620077d4f0 2023-07-14T11:16:48.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f1d2000f040 con 0x55620077d4f0 2023-07-14T11:16:48.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=46) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1d240b83f0 con 0x55620077d4f0 2023-07-14T11:16:48.347 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d43756b80 10 client.26309 fetch_fsmap learned FSMap version 46 2023-07-14T11:16:48.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=46,monmap=4+}) v3 -- 0x5562004fb2b0 con 0x55620077d4f0 2023-07-14T11:16:48.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.336+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1d2000f040 con 0x55620077d4f0 2023-07-14T11:16:48.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 46) v1 ==== 7502+0+0 (secure 0 0 0) 0x7f1d240a2090 con 0x55620077d4f0 2023-07-14T11:16:48.348 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 10 client.26309 fetch_fsmap finished waiting for FSMap version 46 2023-07-14T11:16:48.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 10 client.26309 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:16:48.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 20 client.26309 populate_metadata read hostname 'smithi040' 2023-07-14T11:16:48.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5562007ec5a0 0x5562007ee980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:16:48.349 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 4 client.26309 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:16:48.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55620068a050 con 0x5562007ec5a0 2023-07-14T11:16:48.350 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.337+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5562007ec5a0 0x5562007ee980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:16:48.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d117fa700 10 client.26309 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:48.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d2d39a700 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5562007ec5a0 0x5562007ee980 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=513ba1357ebc53e3 server_cookie=be4767f00ad6ed64 in_seq=0 out_seq=0 2023-07-14T11:16:48.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d117fa700 1 -- 172.21.15.40:0/3134409807 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55620068a050 con 0x5562007ec5a0 2023-07-14T11:16:48.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d117fa700 10 client.26309 handle_command_reply: tid=0 2023-07-14T11:16:48.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 client.26309 shutdown 2023-07-14T11:16:48.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 msgr2=0x55620078e830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f1d24000c00 tx=0x7f1d24098680 comp rx=0 tx=0).stop 2023-07-14T11:16:48.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 shutdown_connections 2023-07-14T11:16:48.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5562007ec5a0 0x5562007ee980 crc :-1 s=READY pgs=90 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.353 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x556200787d30 0x55620078d180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55620077d4f0 0x55620078e830 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d117fa700 0 client.26309 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:16:48.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 --2- 172.21.15.40:0/3134409807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55620077de00 0x55620078cc40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.338+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 >> 172.21.15.40:0/3134409807 conn(0x556200772ff0 msgr2=0x55620077fc50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 shutdown_connections 2023-07-14T11:16:48.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d43756b80 1 -- 172.21.15.40:0/3134409807 wait complete. 2023-07-14T11:16:48.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d43756b80 20 client.26309 trim_cache size 0 max 16384 2023-07-14T11:16:48.356 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:16:48.356 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:16:48.356 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:16:48.357 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:16:48.357 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:16:48.357 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:16:48.357 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:16:48.358 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:16:48.358 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:16:48.358 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:16:48.358 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 msgr2=0x7f1d140a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 0x7f1d140a3810 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f1d280fff50 tx=0x7f1d18005fb0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 msgr2=0x7f1d2819da50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:16:48.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f1d2400eb30 tx=0x7f1d2400ee40 comp rx=0 tx=0).stop 2023-07-14T11:16:48.359 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 shutdown_connections 2023-07-14T11:16:48.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1d140a1360 0x7f1d140a3810 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d2810c3b0 0x7f1d28197ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1d280fe800 0x7f1d2819da50 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 --2- 172.21.15.40:0/3768311089 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1d280ff110 0x7f1d2819df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:16:48.360 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.339+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 >> 172.21.15.40:0/3768311089 conn(0x7f1d280745d0 msgr2=0x7f1d280fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:16:48.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.340+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 shutdown_connections 2023-07-14T11:16:48.361 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:16:48.340+0000 7f1d2eb9d700 1 -- 172.21.15.40:0/3768311089 wait complete. 2023-07-14T11:16:48.362 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:16:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:49 smithi130 ceph-mon[129405]: pgmap v143: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:49 smithi040 ceph-mon[97954]: pgmap v143: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:50.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:49 smithi045 ceph-mon[128791]: pgmap v143: 129 pgs: 129 active+clean; 6.6 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 10 MiB/s rd, 3.0 MiB/s wr, 272 op/s 2023-07-14T11:16:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: active paths [/] 2023-07-14T11:16:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:50 smithi130 ceph-mon[129405]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: active paths [/] 2023-07-14T11:16:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:50 smithi040 ceph-mon[97954]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: active paths [/] 2023-07-14T11:16:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:50 smithi045 ceph-mon[128791]: scrub summary: idle+waiting paths [/] 2023-07-14T11:16:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:51 smithi130 ceph-mon[129405]: pgmap v144: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 MiB/s rd, 10 MiB/s wr, 286 op/s 2023-07-14T11:16:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:51 smithi040 ceph-mon[97954]: pgmap v144: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 MiB/s rd, 10 MiB/s wr, 286 op/s 2023-07-14T11:16:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:51 smithi045 ceph-mon[128791]: pgmap v144: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 22 MiB/s rd, 10 MiB/s wr, 286 op/s 2023-07-14T11:16:54.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:53 smithi130 ceph-mon[129405]: pgmap v145: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 10 MiB/s wr, 277 op/s 2023-07-14T11:16:54.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:53 smithi040 ceph-mon[97954]: pgmap v145: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 10 MiB/s wr, 277 op/s 2023-07-14T11:16:54.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:53 smithi045 ceph-mon[128791]: pgmap v145: 129 pgs: 129 active+clean; 6.7 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 10 MiB/s wr, 277 op/s 2023-07-14T11:16:55.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:55 smithi130 ceph-mon[129405]: pgmap v146: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 25 MiB/s wr, 294 op/s 2023-07-14T11:16:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:55 smithi040 ceph-mon[97954]: pgmap v146: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 25 MiB/s wr, 294 op/s 2023-07-14T11:16:55.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:55 smithi045 ceph-mon[128791]: pgmap v146: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 MiB/s rd, 25 MiB/s wr, 294 op/s 2023-07-14T11:16:58.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:57 smithi130 ceph-mon[129405]: pgmap v147: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:16:58.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:57 smithi040 ceph-mon[97954]: pgmap v147: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:16:58.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:57 smithi045 ceph-mon[128791]: pgmap v147: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:17:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:16:59 smithi130 ceph-mon[129405]: pgmap v148: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:17:00.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:16:59 smithi040 ceph-mon[97954]: pgmap v148: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:17:00.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:16:59 smithi045 ceph-mon[128791]: pgmap v148: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 30 op/s 2023-07-14T11:17:02.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:01 smithi130 ceph-mon[129405]: pgmap v149: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 37 op/s 2023-07-14T11:17:02.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:01 smithi040 ceph-mon[97954]: pgmap v149: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 37 op/s 2023-07-14T11:17:02.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:01 smithi045 ceph-mon[128791]: pgmap v149: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 12 MiB/s rd, 22 MiB/s wr, 37 op/s 2023-07-14T11:17:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: 973 slow requests, 5 included below; oldest blocked for > 183.521234 secs 2023-07-14T11:17:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: slow request 183.221582 seconds old, received at 2023-07-14T11:13:58.402232+0000: client_request(mds.1:948 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: slow request 183.190394 seconds old, received at 2023-07-14T11:13:58.433419+0000: client_request(mds.1:980 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: slow request 183.147883 seconds old, received at 2023-07-14T11:13:58.475930+0000: client_request(mds.1:1012 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: slow request 183.114251 seconds old, received at 2023-07-14T11:13:58.509562+0000: client_request(mds.1:1044 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:02 smithi130 ceph-mon[129405]: slow request 183.081030 seconds old, received at 2023-07-14T11:13:58.542783+0000: client_request(mds.1:1076 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: 973 slow requests, 5 included below; oldest blocked for > 183.521234 secs 2023-07-14T11:17:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: slow request 183.221582 seconds old, received at 2023-07-14T11:13:58.402232+0000: client_request(mds.1:948 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: slow request 183.190394 seconds old, received at 2023-07-14T11:13:58.433419+0000: client_request(mds.1:980 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: slow request 183.147883 seconds old, received at 2023-07-14T11:13:58.475930+0000: client_request(mds.1:1012 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: slow request 183.114251 seconds old, received at 2023-07-14T11:13:58.509562+0000: client_request(mds.1:1044 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:02 smithi040 ceph-mon[97954]: slow request 183.081030 seconds old, received at 2023-07-14T11:13:58.542783+0000: client_request(mds.1:1076 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: 973 slow requests, 5 included below; oldest blocked for > 183.521234 secs 2023-07-14T11:17:03.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: slow request 183.221582 seconds old, received at 2023-07-14T11:13:58.402232+0000: client_request(mds.1:948 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: slow request 183.190394 seconds old, received at 2023-07-14T11:13:58.433419+0000: client_request(mds.1:980 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: slow request 183.147883 seconds old, received at 2023-07-14T11:13:58.475930+0000: client_request(mds.1:1012 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: slow request 183.114251 seconds old, received at 2023-07-14T11:13:58.509562+0000: client_request(mds.1:1044 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:03.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:02 smithi045 ceph-mon[128791]: slow request 183.081030 seconds old, received at 2023-07-14T11:13:58.542783+0000: client_request(mds.1:1076 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: pgmap v150: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 15 MiB/s wr, 23 op/s 2023-07-14T11:17:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: 1645 slow requests, 5 included below; oldest blocked for > 182.159096 secs 2023-07-14T11:17:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: slow request 182.088718 seconds old, received at 2023-07-14T11:14:01.003934+0000: peer_request:mds.1:933 currently dispatched 2023-07-14T11:17:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: slow request 182.082946 seconds old, received at 2023-07-14T11:14:01.009706+0000: peer_request:mds.1:965 currently dispatched 2023-07-14T11:17:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: slow request 182.077112 seconds old, received at 2023-07-14T11:14:01.015541+0000: peer_request:mds.1:997 currently dispatched 2023-07-14T11:17:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: slow request 182.071263 seconds old, received at 2023-07-14T11:14:01.021390+0000: peer_request:mds.1:1029 currently dispatched 2023-07-14T11:17:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:03 smithi130 ceph-mon[129405]: slow request 182.065553 seconds old, received at 2023-07-14T11:14:01.027099+0000: peer_request:mds.1:1061 currently dispatched 2023-07-14T11:17:04.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: pgmap v150: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 15 MiB/s wr, 23 op/s 2023-07-14T11:17:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: 1645 slow requests, 5 included below; oldest blocked for > 182.159096 secs 2023-07-14T11:17:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: slow request 182.088718 seconds old, received at 2023-07-14T11:14:01.003934+0000: peer_request:mds.1:933 currently dispatched 2023-07-14T11:17:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: slow request 182.082946 seconds old, received at 2023-07-14T11:14:01.009706+0000: peer_request:mds.1:965 currently dispatched 2023-07-14T11:17:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: slow request 182.077112 seconds old, received at 2023-07-14T11:14:01.015541+0000: peer_request:mds.1:997 currently dispatched 2023-07-14T11:17:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: slow request 182.071263 seconds old, received at 2023-07-14T11:14:01.021390+0000: peer_request:mds.1:1029 currently dispatched 2023-07-14T11:17:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:03 smithi040 ceph-mon[97954]: slow request 182.065553 seconds old, received at 2023-07-14T11:14:01.027099+0000: peer_request:mds.1:1061 currently dispatched 2023-07-14T11:17:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: pgmap v150: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 29 KiB/s rd, 15 MiB/s wr, 23 op/s 2023-07-14T11:17:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: 1645 slow requests, 5 included below; oldest blocked for > 182.159096 secs 2023-07-14T11:17:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: slow request 182.088718 seconds old, received at 2023-07-14T11:14:01.003934+0000: peer_request:mds.1:933 currently dispatched 2023-07-14T11:17:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: slow request 182.082946 seconds old, received at 2023-07-14T11:14:01.009706+0000: peer_request:mds.1:965 currently dispatched 2023-07-14T11:17:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: slow request 182.077112 seconds old, received at 2023-07-14T11:14:01.015541+0000: peer_request:mds.1:997 currently dispatched 2023-07-14T11:17:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: slow request 182.071263 seconds old, received at 2023-07-14T11:14:01.021390+0000: peer_request:mds.1:1029 currently dispatched 2023-07-14T11:17:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:03 smithi045 ceph-mon[128791]: slow request 182.065553 seconds old, received at 2023-07-14T11:14:01.027099+0000: peer_request:mds.1:1061 currently dispatched 2023-07-14T11:17:05.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:04 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:04 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:05.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:04 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:04 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:05.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:04 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:04 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:06.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:05 smithi130 ceph-mon[129405]: pgmap v151: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 15 MiB/s wr, 41 op/s 2023-07-14T11:17:06.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:05 smithi040 ceph-mon[97954]: pgmap v151: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 15 MiB/s wr, 41 op/s 2023-07-14T11:17:06.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:05 smithi045 ceph-mon[128791]: pgmap v151: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 42 KiB/s rd, 15 MiB/s wr, 41 op/s 2023-07-14T11:17:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: pgmap v152: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: 1945 slow requests, 5 included below; oldest blocked for > 188.564984 secs 2023-07-14T11:17:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: slow request 188.097208 seconds old, received at 2023-07-14T11:13:58.570355+0000: client_request(mds.1:1108 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: slow request 188.065040 seconds old, received at 2023-07-14T11:13:58.602523+0000: client_request(mds.1:1140 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: slow request 188.032859 seconds old, received at 2023-07-14T11:13:58.634704+0000: client_request(mds.1:1172 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: slow request 188.000443 seconds old, received at 2023-07-14T11:13:58.667120+0000: client_request(mds.1:1204 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:07 smithi130 ceph-mon[129405]: slow request 187.971318 seconds old, received at 2023-07-14T11:13:58.696245+0000: client_request(mds.1:1236 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: pgmap v152: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: 1945 slow requests, 5 included below; oldest blocked for > 188.564984 secs 2023-07-14T11:17:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: slow request 188.097208 seconds old, received at 2023-07-14T11:13:58.570355+0000: client_request(mds.1:1108 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: slow request 188.065040 seconds old, received at 2023-07-14T11:13:58.602523+0000: client_request(mds.1:1140 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: slow request 188.032859 seconds old, received at 2023-07-14T11:13:58.634704+0000: client_request(mds.1:1172 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: slow request 188.000443 seconds old, received at 2023-07-14T11:13:58.667120+0000: client_request(mds.1:1204 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:07 smithi040 ceph-mon[97954]: slow request 187.971318 seconds old, received at 2023-07-14T11:13:58.696245+0000: client_request(mds.1:1236 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: pgmap v152: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: 1945 slow requests, 5 included below; oldest blocked for > 188.564984 secs 2023-07-14T11:17:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: slow request 188.097208 seconds old, received at 2023-07-14T11:13:58.570355+0000: client_request(mds.1:1108 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: slow request 188.065040 seconds old, received at 2023-07-14T11:13:58.602523+0000: client_request(mds.1:1140 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: slow request 188.032859 seconds old, received at 2023-07-14T11:13:58.634704+0000: client_request(mds.1:1172 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: slow request 188.000443 seconds old, received at 2023-07-14T11:13:58.667120+0000: client_request(mds.1:1204 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:07 smithi045 ceph-mon[128791]: slow request 187.971318 seconds old, received at 2023-07-14T11:13:58.696245+0000: client_request(mds.1:1236 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:09.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: 1638 slow requests, 5 included below; oldest blocked for > 187.159202 secs 2023-07-14T11:17:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: slow request 187.057390 seconds old, received at 2023-07-14T11:14:01.035368+0000: peer_request:mds.1:1093 currently dispatched 2023-07-14T11:17:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: slow request 187.051600 seconds old, received at 2023-07-14T11:14:01.041158+0000: peer_request:mds.1:1125 currently dispatched 2023-07-14T11:17:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: slow request 187.041940 seconds old, received at 2023-07-14T11:14:01.050818+0000: peer_request:mds.1:1157 currently dispatched 2023-07-14T11:17:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: slow request 187.035497 seconds old, received at 2023-07-14T11:14:01.057261+0000: peer_request:mds.1:1189 currently dispatched 2023-07-14T11:17:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:08 smithi130 ceph-mon[129405]: slow request 187.028988 seconds old, received at 2023-07-14T11:14:01.063770+0000: peer_request:mds.1:1221 currently dispatched 2023-07-14T11:17:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: 1638 slow requests, 5 included below; oldest blocked for > 187.159202 secs 2023-07-14T11:17:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: slow request 187.057390 seconds old, received at 2023-07-14T11:14:01.035368+0000: peer_request:mds.1:1093 currently dispatched 2023-07-14T11:17:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: slow request 187.051600 seconds old, received at 2023-07-14T11:14:01.041158+0000: peer_request:mds.1:1125 currently dispatched 2023-07-14T11:17:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: slow request 187.041940 seconds old, received at 2023-07-14T11:14:01.050818+0000: peer_request:mds.1:1157 currently dispatched 2023-07-14T11:17:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: slow request 187.035497 seconds old, received at 2023-07-14T11:14:01.057261+0000: peer_request:mds.1:1189 currently dispatched 2023-07-14T11:17:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:08 smithi040 ceph-mon[97954]: slow request 187.028988 seconds old, received at 2023-07-14T11:14:01.063770+0000: peer_request:mds.1:1221 currently dispatched 2023-07-14T11:17:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: 1638 slow requests, 5 included below; oldest blocked for > 187.159202 secs 2023-07-14T11:17:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: slow request 187.057390 seconds old, received at 2023-07-14T11:14:01.035368+0000: peer_request:mds.1:1093 currently dispatched 2023-07-14T11:17:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: slow request 187.051600 seconds old, received at 2023-07-14T11:14:01.041158+0000: peer_request:mds.1:1125 currently dispatched 2023-07-14T11:17:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: slow request 187.041940 seconds old, received at 2023-07-14T11:14:01.050818+0000: peer_request:mds.1:1157 currently dispatched 2023-07-14T11:17:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: slow request 187.035497 seconds old, received at 2023-07-14T11:14:01.057261+0000: peer_request:mds.1:1189 currently dispatched 2023-07-14T11:17:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:08 smithi045 ceph-mon[128791]: slow request 187.028988 seconds old, received at 2023-07-14T11:14:01.063770+0000: peer_request:mds.1:1221 currently dispatched 2023-07-14T11:17:10.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:09 smithi130 ceph-mon[129405]: pgmap v153: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:09 smithi130 ceph-mon[129405]: Health check update: 2 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:09 smithi040 ceph-mon[97954]: pgmap v153: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:09 smithi040 ceph-mon[97954]: Health check update: 2 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:10.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:09 smithi045 ceph-mon[128791]: pgmap v153: 129 pgs: 129 active+clean; 6.9 GiB data, 14 GiB used, 1.0 TiB / 1.0 TiB avail; 20 KiB/s rd, 31 KiB/s wr, 24 op/s 2023-07-14T11:17:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:09 smithi045 ceph-mon[128791]: Health check update: 2 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:17:12.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:11 smithi130 ceph-mon[129405]: pgmap v154: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 62 op/s 2023-07-14T11:17:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:11 smithi040 ceph-mon[97954]: pgmap v154: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 62 op/s 2023-07-14T11:17:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:12.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:11 smithi045 ceph-mon[128791]: pgmap v154: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 62 op/s 2023-07-14T11:17:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: 1936 slow requests, 5 included below; oldest blocked for > 193.660491 secs 2023-07-14T11:17:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: slow request 193.027100 seconds old, received at 2023-07-14T11:13:58.735970+0000: client_request(mds.1:1268 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: slow request 192.995350 seconds old, received at 2023-07-14T11:13:58.767721+0000: client_request(mds.1:1300 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: slow request 192.960060 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: slow request 192.925614 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:12 smithi130 ceph-mon[129405]: slow request 192.895067 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: 1936 slow requests, 5 included below; oldest blocked for > 193.660491 secs 2023-07-14T11:17:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: slow request 193.027100 seconds old, received at 2023-07-14T11:13:58.735970+0000: client_request(mds.1:1268 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: slow request 192.995350 seconds old, received at 2023-07-14T11:13:58.767721+0000: client_request(mds.1:1300 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: slow request 192.960060 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: slow request 192.925614 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:12 smithi040 ceph-mon[97954]: slow request 192.895067 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: 1936 slow requests, 5 included below; oldest blocked for > 193.660491 secs 2023-07-14T11:17:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: slow request 193.027100 seconds old, received at 2023-07-14T11:13:58.735970+0000: client_request(mds.1:1268 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: slow request 192.995350 seconds old, received at 2023-07-14T11:13:58.767721+0000: client_request(mds.1:1300 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: slow request 192.960060 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: slow request 192.925614 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:12 smithi045 ceph-mon[128791]: slow request 192.895067 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:14.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: pgmap v155: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 2.1 MiB/s wr, 55 op/s 2023-07-14T11:17:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: 1933 slow requests, 5 included below; oldest blocked for > 192.159367 secs 2023-07-14T11:17:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: slow request 192.020273 seconds old, received at 2023-07-14T11:14:01.072650+0000: peer_request:mds.1:1253 currently dispatched 2023-07-14T11:17:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: slow request 192.014575 seconds old, received at 2023-07-14T11:14:01.078349+0000: peer_request:mds.1:1285 currently dispatched 2023-07-14T11:17:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: slow request 192.006570 seconds old, received at 2023-07-14T11:14:01.086353+0000: peer_request:mds.1:1317 currently dispatched 2023-07-14T11:17:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: slow request 191.996577 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:17:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:13 smithi130 ceph-mon[129405]: slow request 191.986559 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:17:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: pgmap v155: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 2.1 MiB/s wr, 55 op/s 2023-07-14T11:17:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: 1933 slow requests, 5 included below; oldest blocked for > 192.159367 secs 2023-07-14T11:17:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: slow request 192.020273 seconds old, received at 2023-07-14T11:14:01.072650+0000: peer_request:mds.1:1253 currently dispatched 2023-07-14T11:17:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: slow request 192.014575 seconds old, received at 2023-07-14T11:14:01.078349+0000: peer_request:mds.1:1285 currently dispatched 2023-07-14T11:17:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: slow request 192.006570 seconds old, received at 2023-07-14T11:14:01.086353+0000: peer_request:mds.1:1317 currently dispatched 2023-07-14T11:17:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: slow request 191.996577 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:17:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:13 smithi040 ceph-mon[97954]: slow request 191.986559 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:17:14.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: pgmap v155: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 2.1 MiB/s wr, 55 op/s 2023-07-14T11:17:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: 1933 slow requests, 5 included below; oldest blocked for > 192.159367 secs 2023-07-14T11:17:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: slow request 192.020273 seconds old, received at 2023-07-14T11:14:01.072650+0000: peer_request:mds.1:1253 currently dispatched 2023-07-14T11:17:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: slow request 192.014575 seconds old, received at 2023-07-14T11:14:01.078349+0000: peer_request:mds.1:1285 currently dispatched 2023-07-14T11:17:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: slow request 192.006570 seconds old, received at 2023-07-14T11:14:01.086353+0000: peer_request:mds.1:1317 currently dispatched 2023-07-14T11:17:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: slow request 191.996577 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:17:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:13 smithi045 ceph-mon[128791]: slow request 191.986559 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:17:16.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:15 smithi130 ceph-mon[129405]: pgmap v156: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 2.1 MiB/s wr, 61 op/s 2023-07-14T11:17:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:16.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:15 smithi040 ceph-mon[97954]: pgmap v156: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 2.1 MiB/s wr, 61 op/s 2023-07-14T11:17:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:16.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:15 smithi045 ceph-mon[128791]: pgmap v156: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 56 KiB/s rd, 2.1 MiB/s wr, 61 op/s 2023-07-14T11:17:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:17.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:17 smithi130 ceph-mon[129405]: pgmap v157: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:17.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:17 smithi040 ceph-mon[97954]: pgmap v157: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:17.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:17 smithi045 ceph-mon[128791]: pgmap v157: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:18.363 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:17:18.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 -- 172.21.15.40:0/2549614672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580fe800 msgr2=0x7faf580febd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2549614672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580fe800 0x7faf580febd0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7faf48009990 tx=0x7faf48009ca0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 -- 172.21.15.40:0/2549614672 shutdown_connections 2023-07-14T11:17:18.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2549614672 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 0x7faf5810e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2549614672 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580fe800 0x7faf580febd0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2549614672 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580ff110 0x7faf5810be70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 -- 172.21.15.40:0/2549614672 >> 172.21.15.40:0/2549614672 conn(0x7faf580745d0 msgr2=0x7faf580749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 -- 172.21.15.40:0/2549614672 shutdown_connections 2023-07-14T11:17:18.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.579+0000 7faf5d39d700 1 -- 172.21.15.40:0/2549614672 wait complete. 2023-07-14T11:17:18.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.581+0000 7faf5d39d700 1 Processor -- start 2023-07-14T11:17:18.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.581+0000 7faf5d39d700 1 -- start start 2023-07-14T11:17:18.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 0x7faf5819d9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 0x7faf58197a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf58198080 con 0x7faf5810c3b0 2023-07-14T11:17:18.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf581981f0 con 0x7faf580ff110 2023-07-14T11:17:18.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5d39d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf581983b0 con 0x7faf580fe800 2023-07-14T11:17:18.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38240/0 (socket says 172.21.15.40:38240) 2023-07-14T11:17:18.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 0x7faf5819d9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf577fe700 1 -- 172.21.15.40:0/2954216226 learned_addr learned my addr 172.21.15.40:0/2954216226 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:18.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 0x7faf58197a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 -- 172.21.15.40:0/2954216226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 msgr2=0x7faf5819dee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 -- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 msgr2=0x7faf58197a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 0x7faf58197a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf57fff700 1 -- 172.21.15.40:0/2954216226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf480095f0 con 0x7faf580fe800 2023-07-14T11:17:18.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.582+0000 7faf577fe700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:17:18.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf57fff700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 0x7faf5819d9a0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7faf48000c00 tx=0x7faf48005740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf557fa700 1 -- 172.21.15.40:0/2954216226 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf4801d070 con 0x7faf580fe800 2023-07-14T11:17:18.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf557fa700 1 -- 172.21.15.40:0/2954216226 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faf48021410 con 0x7faf580fe800 2023-07-14T11:17:18.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faf58198630 con 0x7faf580fe800 2023-07-14T11:17:18.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf557fa700 1 -- 172.21.15.40:0/2954216226 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf48017650 con 0x7faf580fe800 2023-07-14T11:17:18.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.583+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf581a4990 con 0x7faf580fe800 2023-07-14T11:17:18.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.586+0000 7faf557fa700 1 -- 172.21.15.40:0/2954216226 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7faf48021580 con 0x7faf580fe800 2023-07-14T11:17:18.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.588+0000 7faf71e29b80 1 Processor -- start 2023-07-14T11:17:18.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.588+0000 7faf71e29b80 1 -- start start 2023-07-14T11:17:18.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39748c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f3981110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f3879cb0 con 0x55b0f3974e00 2023-07-14T11:17:18.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f38cca40 con 0x55b0f397ed30 2023-07-14T11:17:18.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf71e29b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f38c5ed0 con 0x55b0f39744f0 2023-07-14T11:17:18.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf5cb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39748c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf5cb9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39748c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57800/0 (socket says 172.21.15.40:57800) 2023-07-14T11:17:18.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f3981110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf5cb9c700 1 -- 172.21.15.40:0/3256111515 learned_addr learned my addr 172.21.15.40:0/3256111515 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:18.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf557fa700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 0x7faf400a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf57fff700 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf557fa700 1 -- 172.21.15.40:0/2954216226 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7faf480ce380 con 0x7faf580fe800 2023-07-14T11:17:18.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 0x7faf400a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.589+0000 7faf577fe700 1 -- 172.21.15.40:0/3256111515 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 msgr2=0x55b0f39748c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf577fe700 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39748c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf577fe700 1 -- 172.21.15.40:0/3256111515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 msgr2=0x55b0f397e7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf577fe700 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397e7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf577fe700 1 -- 172.21.15.40:0/3256111515 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b0f36f3de0 con 0x55b0f397ed30 2023-07-14T11:17:18.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf577fe700 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f3981110 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7faf4c005950 tx=0x7faf4c011820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6396bd2aec62a62 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf3e7fc700 1 -- 172.21.15.40:0/3256111515 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf4c004a30 con 0x55b0f397ed30 2023-07-14T11:17:18.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf3e7fc700 1 -- 172.21.15.40:0/3256111515 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faf4c0054e0 con 0x55b0f397ed30 2023-07-14T11:17:18.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.590+0000 7faf3e7fc700 1 -- 172.21.15.40:0/3256111515 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf4c019930 con 0x55b0f397ed30 2023-07-14T11:17:18.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 0x7faf400a3890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7faf50006070 tx=0x7faf50004b40 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 -- 172.21.15.40:0/3256111515 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 msgr2=0x55b0f3981110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f3981110 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7faf4c005950 tx=0x7faf4c011820 comp rx=0 tx=0).stop 2023-07-14T11:17:18.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 -- 172.21.15.40:0/3256111515 shutdown_connections 2023-07-14T11:17:18.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f3981110 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39748c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3256111515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397e7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 -- 172.21.15.40:0/3256111515 >> 172.21.15.40:0/3256111515 conn(0x55b0f3969ff0 msgr2=0x55b0f396f930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 -- 172.21.15.40:0/3256111515 shutdown_connections 2023-07-14T11:17:18.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.591+0000 7faf71e29b80 1 -- 172.21.15.40:0/3256111515 wait complete. 2023-07-14T11:17:18.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 -- start start 2023-07-14T11:17:18.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 Processor -- start 2023-07-14T11:17:18.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397b8d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f397be70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f3865ac0 con 0x55b0f3974e00 2023-07-14T11:17:18.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f38657c0 con 0x55b0f397ed30 2023-07-14T11:17:18.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.592+0000 7faf71e29b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f3749920 con 0x55b0f39744f0 2023-07-14T11:17:18.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf71e29b80 10 client.0 _ll_register_callbacks cb 0x55b0f3863760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:17:18.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57804/0 (socket says 172.21.15.40:57804) 2023-07-14T11:17:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 -- 172.21.15.40:0/1752471766 learned_addr learned my addr 172.21.15.40:0/1752471766 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f397be70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf577fe700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397b8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 -- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 msgr2=0x55b0f397be70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f397be70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 -- 172.21.15.40:0/1752471766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 msgr2=0x55b0f397b8d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397b8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 -- 172.21.15.40:0/1752471766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf4c009d60 con 0x55b0f39744f0 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.593+0000 7faf57fff700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7faf48097be0 tx=0x7faf48009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=37084611243ff14f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:17:18.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf480cbbf0 con 0x55b0f39744f0 2023-07-14T11:17:18.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7faf4809b170 con 0x55b0f39744f0 2023-07-14T11:17:18.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55b0f381f4f0 con 0x55b0f39744f0 2023-07-14T11:17:18.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=53) v2 ==== 24+0+0 (secure 0 0 0) 0x7faf4809f1a0 con 0x55b0f39744f0 2023-07-14T11:17:18.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf71e29b80 10 client.26284 fetch_fsmap learned FSMap version 53 2023-07-14T11:17:18.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=53,monmap=4+}) v3 -- 0x55b0f36dcc60 con 0x55b0f39744f0 2023-07-14T11:17:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55b0f381f4f0 con 0x55b0f39744f0 2023-07-14T11:17:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 53) v1 ==== 7502+0+0 (secure 0 0 0) 0x7faf480a30a0 con 0x55b0f39744f0 2023-07-14T11:17:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.594+0000 7faf71e29b80 10 client.26284 fetch_fsmap finished waiting for FSMap version 53 2023-07-14T11:17:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf71e29b80 10 client.26284 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:17:18.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf71e29b80 20 client.26284 populate_metadata read hostname 'smithi040' 2023-07-14T11:17:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f398e240 0x55b0f3990650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf71e29b80 4 client.26284 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:17:18.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55b0f38beeb0 con 0x55b0f398e240 2023-07-14T11:17:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf577fe700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f398e240 0x55b0f3990650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf377fe700 10 client.26284 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.595+0000 7faf577fe700 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f398e240 0x55b0f3990650 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=defccfbbd364b590 server_cookie=53ed7d3f49bf0f8b in_seq=0 out_seq=0 2023-07-14T11:17:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.596+0000 7faf377fe700 1 -- 172.21.15.40:0/1752471766 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55b0f38beeb0 con 0x55b0f398e240 2023-07-14T11:17:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.596+0000 7faf377fe700 10 client.26284 handle_command_reply: tid=0 2023-07-14T11:17:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.596+0000 7faf71e29b80 1 client.26284 shutdown 2023-07-14T11:17:18.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 msgr2=0x55b0f397e650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7faf48097be0 tx=0x7faf48009ca0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 shutdown_connections 2023-07-14T11:17:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f398e240 0x55b0f3990650 crc :-1 s=READY pgs=91 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f397ed30 0x55b0f397be70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf377fe700 0 client.26284 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:18.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f397e650 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1752471766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f397b8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 >> 172.21.15.40:0/1752471766 conn(0x55b0f3969ff0 msgr2=0x55b0f396f4c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 shutdown_connections 2023-07-14T11:17:18.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 1 -- 172.21.15.40:0/1752471766 wait complete. 2023-07-14T11:17:18.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.597+0000 7faf71e29b80 20 client.26284 trim_cache size 0 max 16384 2023-07-14T11:17:18.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.632+0000 7faf71e29b80 1 Processor -- start 2023-07-14T11:17:18.635 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 -- start start 2023-07-14T11:17:18.636 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 0x55b0f397e1f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f3974e00 0x55b0f3978b80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f39767e0 con 0x55b0f39744f0 2023-07-14T11:17:18.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f3975980 con 0x55b0f3969ff0 2023-07-14T11:17:18.637 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf71e29b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f39761d0 con 0x55b0f3974e00 2023-07-14T11:17:18.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38268/0 (socket says 172.21.15.40:38268) 2023-07-14T11:17:18.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.633+0000 7faf57fff700 1 -- 172.21.15.40:0/1830723251 learned_addr learned my addr 172.21.15.40:0/1830723251 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:18.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f3974e00 0x55b0f3978b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.638 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf577fe700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 0x55b0f397e1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 -- 172.21.15.40:0/1830723251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f3974e00 msgr2=0x55b0f3978b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f3974e00 0x55b0f3978b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 -- 172.21.15.40:0/1830723251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 msgr2=0x55b0f397e1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 0x55b0f397e1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.639 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 -- 172.21.15.40:0/1830723251 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55b0f371a250 con 0x55b0f3969ff0 2023-07-14T11:17:18.640 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf577fe700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 0x55b0f397e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:17:18.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf57fff700 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7faf48009990 tx=0x7faf4809a9e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=409e413929204e5b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf3effd700 1 -- 172.21.15.40:0/1830723251 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf4809a660 con 0x55b0f3969ff0 2023-07-14T11:17:18.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf3effd700 1 -- 172.21.15.40:0/1830723251 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faf480cbbf0 con 0x55b0f3969ff0 2023-07-14T11:17:18.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.634+0000 7faf3effd700 1 -- 172.21.15.40:0/1830723251 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf480970b0 con 0x55b0f3969ff0 2023-07-14T11:17:18.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 -- 172.21.15.40:0/1830723251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 msgr2=0x55b0f3985870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7faf48009990 tx=0x7faf4809a9e0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 -- 172.21.15.40:0/1830723251 shutdown_connections 2023-07-14T11:17:18.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f3974e00 0x55b0f3978b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.643 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f3985870 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 --2- 172.21.15.40:0/1830723251 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f39744f0 0x55b0f397e1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.635+0000 7faf71e29b80 1 -- 172.21.15.40:0/1830723251 >> 172.21.15.40:0/1830723251 conn(0x55b0f39813e0 msgr2=0x55b0f398a010 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.636+0000 7faf71e29b80 1 -- 172.21.15.40:0/1830723251 shutdown_connections 2023-07-14T11:17:18.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.636+0000 7faf71e29b80 1 -- 172.21.15.40:0/1830723251 wait complete. 2023-07-14T11:17:18.644 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.636+0000 7faf71e29b80 1 -- start start 2023-07-14T11:17:18.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.636+0000 7faf71e29b80 1 Processor -- start 2023-07-14T11:17:18.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39dc050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f39d6090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.645 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f39d6670 con 0x55b0f3974e00 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f39d67e0 con 0x55b0f3969ff0 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55b0f39d69a0 con 0x55b0f39744f0 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf71e29b80 10 client.0 _ll_register_callbacks cb 0x7faf4809acf0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38274/0 (socket says 172.21.15.40:38274) 2023-07-14T11:17:18.646 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.637+0000 7faf57fff700 1 -- 172.21.15.40:0/3505888245 learned_addr learned my addr 172.21.15.40:0/3505888245 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:18.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf577fe700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39dc050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf5cb9c700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f39d6090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 -- 172.21.15.40:0/3505888245 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 msgr2=0x55b0f39dc050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39dc050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 -- 172.21.15.40:0/3505888245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 msgr2=0x55b0f39d6090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:18.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f39d6090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 -- 172.21.15.40:0/3505888245 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55b0f36dcc60 con 0x55b0f3969ff0 2023-07-14T11:17:18.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf57fff700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7faf48097670 tx=0x7faf480a32d0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9bd0784eb7d59f35 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:18.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf377fe700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:17:18.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7faf4809a7f0 con 0x55b0f3969ff0 2023-07-14T11:17:18.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7faf4c0054e0 con 0x55b0f3969ff0 2023-07-14T11:17:18.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.638+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7faf4809b170 con 0x55b0f3969ff0 2023-07-14T11:17:18.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=53) v2 ==== 24+0+0 (secure 0 0 0) 0x7faf480a3530 con 0x55b0f3969ff0 2023-07-14T11:17:18.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 10 client.26327 fetch_fsmap learned FSMap version 53 2023-07-14T11:17:18.649 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=53,monmap=4+}) v3 -- 0x55b0f36f22b0 con 0x55b0f3969ff0 2023-07-14T11:17:18.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7faf4c0054e0 con 0x55b0f3969ff0 2023-07-14T11:17:18.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 53) v1 ==== 7502+0+0 (secure 0 0 0) 0x7faf4809a0d0 con 0x55b0f3969ff0 2023-07-14T11:17:18.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 10 client.26327 fetch_fsmap finished waiting for FSMap version 53 2023-07-14T11:17:18.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 10 client.26327 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:17:18.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 20 client.26327 populate_metadata read hostname 'smithi040' 2023-07-14T11:17:18.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f397ed30 0x55b0f39e5420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:18.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 4 client.26327 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:17:18.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55b0f3881050 con 0x55b0f397ed30 2023-07-14T11:17:18.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.639+0000 7faf577fe700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f397ed30 0x55b0f39e5420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:18.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf577fe700 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f397ed30 0x55b0f39e5420 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e45a459533b71870 server_cookie=cdb43b4f3075f553 in_seq=0 out_seq=0 2023-07-14T11:17:18.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf377fe700 10 client.26327 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:18.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf377fe700 1 -- 172.21.15.40:0/3505888245 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55b0f3881050 con 0x55b0f397ed30 2023-07-14T11:17:18.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf377fe700 10 client.26327 handle_command_reply: tid=0 2023-07-14T11:17:18.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf71e29b80 1 client.26327 shutdown 2023-07-14T11:17:18.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 msgr2=0x55b0f39dbb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.640+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7faf48097670 tx=0x7faf480a32d0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 shutdown_connections 2023-07-14T11:17:18.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55b0f397ed30 0x55b0f39e5420 crc :-1 s=READY pgs=92 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55b0f3974e00 0x55b0f39d6090 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55b0f3969ff0 0x55b0f39dbb10 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 --2- 172.21.15.40:0/3505888245 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55b0f39744f0 0x55b0f39dc050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf377fe700 0 client.26327 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 >> 172.21.15.40:0/3505888245 conn(0x55b0f39813e0 msgr2=0x55b0f3973440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 shutdown_connections 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 1 -- 172.21.15.40:0/3505888245 wait complete. 2023-07-14T11:17:18.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.641+0000 7faf71e29b80 20 client.26327 trim_cache size 0 max 16384 2023-07-14T11:17:18.656 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:17:18.656 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:17:18.656 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:17:18.657 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:17:18.657 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:17:18.657 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:17:18.657 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:17:18.657 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:17:18.658 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:17:18.658 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:17:18.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 msgr2=0x7faf400a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 0x7faf400a3890 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7faf50006070 tx=0x7faf50004b40 comp rx=0 tx=0).stop 2023-07-14T11:17:18.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 msgr2=0x7faf5819d9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:18.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 0x7faf5819d9a0 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7faf48000c00 tx=0x7faf48005740 comp rx=0 tx=0).stop 2023-07-14T11:17:18.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 shutdown_connections 2023-07-14T11:17:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faf400a13e0 0x7faf400a3890 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf5810c3b0 0x7faf58197a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7faf580fe800 0x7faf5819d9a0 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 --2- 172.21.15.40:0/2954216226 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7faf580ff110 0x7faf5819dee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:18.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 >> 172.21.15.40:0/2954216226 conn(0x7faf580745d0 msgr2=0x7faf580fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:18.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 shutdown_connections 2023-07-14T11:17:18.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:18.642+0000 7faf5d39d700 1 -- 172.21.15.40:0/2954216226 wait complete. 2023-07-14T11:17:18.671 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:17:18.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: 1926 slow requests, 5 included below; oldest blocked for > 198.866609 secs 2023-07-14T11:17:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 198.072153 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 198.037253 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 197.996440 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 197.959568 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 197.926003 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: 1923 slow requests, 5 included below; oldest blocked for > 197.159489 secs 2023-07-14T11:17:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 196.976577 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:17:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 196.966579 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:17:18.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 196.956611 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:17:18.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 196.946530 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:17:18.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:18 smithi130 ceph-mon[129405]: slow request 196.936441 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:17:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: 1926 slow requests, 5 included below; oldest blocked for > 198.866609 secs 2023-07-14T11:17:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 198.072153 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 198.037253 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 197.996440 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 197.959568 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 197.926003 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: 1923 slow requests, 5 included below; oldest blocked for > 197.159489 secs 2023-07-14T11:17:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 196.976577 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:17:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 196.966579 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:17:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 196.956611 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:17:18.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 196.946530 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:17:18.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:18 smithi040 ceph-mon[97954]: slow request 196.936441 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:17:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: 1926 slow requests, 5 included below; oldest blocked for > 198.866609 secs 2023-07-14T11:17:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 198.072153 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 198.037253 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 197.996440 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 197.959568 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 197.926003 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: 1923 slow requests, 5 included below; oldest blocked for > 197.159489 secs 2023-07-14T11:17:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 196.976577 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:17:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 196.966579 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:17:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 196.956611 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:17:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 196.946530 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:17:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:18 smithi045 ceph-mon[128791]: slow request 196.936441 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:17:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:19.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:19 smithi130 ceph-mon[129405]: pgmap v158: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:19.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:19.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:19 smithi040 ceph-mon[97954]: pgmap v158: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:19 smithi045 ceph-mon[128791]: pgmap v158: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 2.1 MiB/s wr, 43 op/s 2023-07-14T11:17:21.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:21.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:21.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:21 smithi130 ceph-mon[129405]: pgmap v159: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 2.1 MiB/s wr, 49 op/s 2023-07-14T11:17:21.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:21.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:21.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:21 smithi040 ceph-mon[97954]: pgmap v159: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 2.1 MiB/s wr, 49 op/s 2023-07-14T11:17:21.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:21.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:21.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:21 smithi045 ceph-mon[128791]: pgmap v159: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 50 KiB/s rd, 2.1 MiB/s wr, 49 op/s 2023-07-14T11:17:22.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:22 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:17:22.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:22 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:17:22.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:22 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:17:23.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: 1915 slow requests, 5 included below; oldest blocked for > 204.096023 secs 2023-07-14T11:17:23.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 203.128451 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 203.095001 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 203.065870 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 203.033304 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 202.981762 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: pgmap v160: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 37 KiB/s wr, 11 op/s 2023-07-14T11:17:23.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: 1913 slow requests, 5 included below; oldest blocked for > 202.159571 secs 2023-07-14T11:17:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 201.926515 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:17:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 201.918278 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:17:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 201.908598 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:17:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 201.898727 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:17:23.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: slow request 201.888704 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:17:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:17:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:17:23.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:23 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:17:23.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: 1915 slow requests, 5 included below; oldest blocked for > 204.096023 secs 2023-07-14T11:17:23.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 203.128451 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 203.095001 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 203.065870 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 203.033304 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 202.981762 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: pgmap v160: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 37 KiB/s wr, 11 op/s 2023-07-14T11:17:23.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: 1913 slow requests, 5 included below; oldest blocked for > 202.159571 secs 2023-07-14T11:17:23.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 201.926515 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:17:23.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 201.918278 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:17:23.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 201.908598 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:17:23.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 201.898727 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:17:23.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: slow request 201.888704 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:17:23.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:17:23.799 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:17:23.800 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:23 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:17:23.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: 1915 slow requests, 5 included below; oldest blocked for > 204.096023 secs 2023-07-14T11:17:23.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 203.128451 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 203.095001 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 203.065870 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 203.033304 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 202.981762 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:23.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: pgmap v160: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 47 KiB/s rd, 37 KiB/s wr, 11 op/s 2023-07-14T11:17:23.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: 1913 slow requests, 5 included below; oldest blocked for > 202.159571 secs 2023-07-14T11:17:23.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 201.926515 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:17:23.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 201.918278 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:17:23.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 201.908598 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:17:23.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 201.898727 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:17:23.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: slow request 201.888704 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:17:23.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:17:23.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:17:23.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:23 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:17:25.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:25.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:25.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:25 smithi130 ceph-mon[129405]: pgmap v161: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:17:25.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:25.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:25.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:25 smithi040 ceph-mon[97954]: pgmap v161: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:17:25.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:25.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:25.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:25 smithi045 ceph-mon[128791]: pgmap v161: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 51 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:17:27.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:27.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:27.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:27 smithi130 ceph-mon[129405]: pgmap v162: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:27.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:27.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:27 smithi040 ceph-mon[97954]: pgmap v162: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:27.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:27.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:27.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:27 smithi045 ceph-mon[128791]: pgmap v162: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:28.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: 1903 slow requests, 5 included below; oldest blocked for > 207.159695 secs 2023-07-14T11:17:28.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: slow request 206.878778 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:17:28.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: slow request 206.868676 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:17:28.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: slow request 206.858638 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:17:28.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: slow request 206.848629 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:17:28.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:28 smithi130 ceph-mon[129405]: slow request 206.838582 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:17:28.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: 1903 slow requests, 5 included below; oldest blocked for > 207.159695 secs 2023-07-14T11:17:28.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: slow request 206.878778 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:17:28.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: slow request 206.868676 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:17:28.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: slow request 206.858638 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:17:28.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: slow request 206.848629 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:17:28.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:28 smithi040 ceph-mon[97954]: slow request 206.838582 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:17:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: 1903 slow requests, 5 included below; oldest blocked for > 207.159695 secs 2023-07-14T11:17:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: slow request 206.878778 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:17:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: slow request 206.868676 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:17:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: slow request 206.858638 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:17:28.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: slow request 206.848629 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:17:28.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:28 smithi045 ceph-mon[128791]: slow request 206.838582 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:17:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: 1904 slow requests, 5 included below; oldest blocked for > 209.744334 secs 2023-07-14T11:17:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: slow request 208.572548 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: slow request 208.546021 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: slow request 208.517788 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: slow request 208.490191 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: slow request 208.463653 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:29.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:29 smithi130 ceph-mon[129405]: pgmap v163: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:29.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: 1904 slow requests, 5 included below; oldest blocked for > 209.744334 secs 2023-07-14T11:17:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: slow request 208.572548 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: slow request 208.546021 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: slow request 208.517788 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: slow request 208.490191 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: slow request 208.463653 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:29.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:29 smithi040 ceph-mon[97954]: pgmap v163: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: 1904 slow requests, 5 included below; oldest blocked for > 209.744334 secs 2023-07-14T11:17:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: slow request 208.572548 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: slow request 208.546021 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: slow request 208.517788 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: slow request 208.490191 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: slow request 208.463653 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:29.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:29 smithi045 ceph-mon[128791]: pgmap v163: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.9 KiB/s rd, 6.3 KiB/s wr, 10 op/s 2023-07-14T11:17:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:31.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:31 smithi045 ceph-mon[128791]: pgmap v164: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 13 KiB/s wr, 16 op/s 2023-07-14T11:17:32.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:31 smithi130 ceph-mon[129405]: pgmap v164: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 13 KiB/s wr, 16 op/s 2023-07-14T11:17:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:31 smithi040 ceph-mon[97954]: pgmap v164: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 18 KiB/s rd, 13 KiB/s wr, 16 op/s 2023-07-14T11:17:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: pgmap v165: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 7.3 KiB/s wr, 10 op/s 2023-07-14T11:17:33.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: 1894 slow requests, 5 included below; oldest blocked for > 212.159790 secs 2023-07-14T11:17:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: slow request 211.832933 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:17:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: slow request 211.827159 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:17:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: slow request 211.821474 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:17:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: slow request 211.815650 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:17:33.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:33 smithi045 ceph-mon[128791]: slow request 211.809812 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:17:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: pgmap v165: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 7.3 KiB/s wr, 10 op/s 2023-07-14T11:17:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: 1894 slow requests, 5 included below; oldest blocked for > 212.159790 secs 2023-07-14T11:17:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: slow request 211.832933 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:17:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: slow request 211.827159 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:17:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: slow request 211.821474 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:17:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: slow request 211.815650 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:17:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:33 smithi130 ceph-mon[129405]: slow request 211.809812 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:17:34.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: pgmap v165: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 11 KiB/s rd, 7.3 KiB/s wr, 10 op/s 2023-07-14T11:17:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: 1894 slow requests, 5 included below; oldest blocked for > 212.159790 secs 2023-07-14T11:17:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: slow request 211.832933 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:17:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: slow request 211.827159 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:17:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: slow request 211.821474 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:17:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: slow request 211.815650 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:17:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:33 smithi040 ceph-mon[97954]: slow request 211.809812 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:17:35.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: 1895 slow requests, 5 included below; oldest blocked for > 214.765338 secs 2023-07-14T11:17:35.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: slow request 213.458388 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: slow request 213.431846 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: slow request 213.405711 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: slow request 213.379545 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:34 smithi130 ceph-mon[129405]: slow request 213.353898 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: 1895 slow requests, 5 included below; oldest blocked for > 214.765338 secs 2023-07-14T11:17:35.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: slow request 213.458388 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: slow request 213.431846 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: slow request 213.405711 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: slow request 213.379545 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:34 smithi040 ceph-mon[97954]: slow request 213.353898 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: 1895 slow requests, 5 included below; oldest blocked for > 214.765338 secs 2023-07-14T11:17:35.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: slow request 213.458388 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: slow request 213.431846 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: slow request 213.405711 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: slow request 213.379545 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:34 smithi045 ceph-mon[128791]: slow request 213.353898 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:36.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:35 smithi130 ceph-mon[129405]: pgmap v166: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 9.7 KiB/s wr, 16 op/s 2023-07-14T11:17:36.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:35 smithi040 ceph-mon[97954]: pgmap v166: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 9.7 KiB/s wr, 16 op/s 2023-07-14T11:17:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:35 smithi045 ceph-mon[128791]: pgmap v166: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 9.7 KiB/s wr, 16 op/s 2023-07-14T11:17:38.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:37 smithi130 ceph-mon[129405]: pgmap v167: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:38.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:37 smithi040 ceph-mon[97954]: pgmap v167: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:38.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:37 smithi045 ceph-mon[128791]: pgmap v167: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: 1883 slow requests, 5 included below; oldest blocked for > 217.159840 secs 2023-07-14T11:17:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: slow request 216.804227 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:17:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: slow request 216.798524 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:17:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: slow request 216.792808 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:17:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: slow request 216.787121 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:17:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:38 smithi130 ceph-mon[129405]: slow request 216.781393 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:17:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: 1883 slow requests, 5 included below; oldest blocked for > 217.159840 secs 2023-07-14T11:17:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: slow request 216.804227 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:17:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: slow request 216.798524 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:17:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: slow request 216.792808 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:17:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: slow request 216.787121 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:17:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:38 smithi040 ceph-mon[97954]: slow request 216.781393 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:17:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: 1883 slow requests, 5 included below; oldest blocked for > 217.159840 secs 2023-07-14T11:17:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: slow request 216.804227 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:17:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: slow request 216.798524 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:17:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: slow request 216.792808 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:17:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: slow request 216.787121 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:17:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:38 smithi045 ceph-mon[128791]: slow request 216.781393 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:17:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: 1884 slow requests, 5 included below; oldest blocked for > 219.883691 secs 2023-07-14T11:17:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: slow request 218.443905 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: slow request 218.416849 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: slow request 218.390302 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: slow request 218.357984 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: slow request 218.332266 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:39 smithi130 ceph-mon[129405]: pgmap v168: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:40.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: 1884 slow requests, 5 included below; oldest blocked for > 219.883691 secs 2023-07-14T11:17:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: slow request 218.443905 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: slow request 218.416849 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: slow request 218.390302 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: slow request 218.357984 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: slow request 218.332266 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:39 smithi040 ceph-mon[97954]: pgmap v168: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: 1884 slow requests, 5 included below; oldest blocked for > 219.883691 secs 2023-07-14T11:17:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: slow request 218.443905 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: slow request 218.416849 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: slow request 218.390302 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: slow request 218.357984 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: slow request 218.332266 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:39 smithi045 ceph-mon[128791]: pgmap v168: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 8.7 KiB/s wr, 11 op/s 2023-07-14T11:17:42.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:41 smithi130 ceph-mon[129405]: pgmap v169: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 9.7 KiB/s wr, 15 op/s 2023-07-14T11:17:42.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:41 smithi040 ceph-mon[97954]: pgmap v169: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 9.7 KiB/s wr, 15 op/s 2023-07-14T11:17:42.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:41 smithi045 ceph-mon[128791]: pgmap v169: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 9.7 KiB/s wr, 15 op/s 2023-07-14T11:17:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: pgmap v170: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.4 KiB/s wr, 9 op/s 2023-07-14T11:17:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: 1872 slow requests, 5 included below; oldest blocked for > 222.159913 secs 2023-07-14T11:17:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: slow request 221.775784 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:17:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: slow request 221.770030 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:17:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: slow request 221.764038 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:17:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: slow request 221.757979 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:17:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:43 smithi130 ceph-mon[129405]: slow request 221.752079 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:17:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: pgmap v170: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.4 KiB/s wr, 9 op/s 2023-07-14T11:17:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: 1872 slow requests, 5 included below; oldest blocked for > 222.159913 secs 2023-07-14T11:17:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: slow request 221.775784 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:17:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: slow request 221.770030 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:17:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: slow request 221.764038 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:17:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: slow request 221.757979 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:17:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:43 smithi040 ceph-mon[97954]: slow request 221.752079 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:17:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: pgmap v170: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 3.4 KiB/s wr, 9 op/s 2023-07-14T11:17:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: 1872 slow requests, 5 included below; oldest blocked for > 222.159913 secs 2023-07-14T11:17:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: slow request 221.775784 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:17:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: slow request 221.770030 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:17:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: slow request 221.764038 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:17:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: slow request 221.757979 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:17:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:43 smithi045 ceph-mon[128791]: slow request 221.752079 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:17:45.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: 1873 slow requests, 5 included below; oldest blocked for > 224.980951 secs 2023-07-14T11:17:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: slow request 223.404253 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: slow request 223.379200 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: slow request 223.351641 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: slow request 223.312451 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:44 smithi130 ceph-mon[129405]: slow request 223.281266 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: 1873 slow requests, 5 included below; oldest blocked for > 224.980951 secs 2023-07-14T11:17:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: slow request 223.404253 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: slow request 223.379200 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: slow request 223.351641 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: slow request 223.312451 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:44 smithi040 ceph-mon[97954]: slow request 223.281266 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: 1873 slow requests, 5 included below; oldest blocked for > 224.980951 secs 2023-07-14T11:17:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: slow request 223.404253 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: slow request 223.379200 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: slow request 223.351641 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: slow request 223.312451 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:44 smithi045 ceph-mon[128791]: slow request 223.281266 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:46.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:45 smithi130 ceph-mon[129405]: pgmap v171: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.3 KiB/s rd, 3.4 KiB/s wr, 13 op/s 2023-07-14T11:17:46.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:45 smithi040 ceph-mon[97954]: pgmap v171: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.3 KiB/s rd, 3.4 KiB/s wr, 13 op/s 2023-07-14T11:17:46.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:45 smithi045 ceph-mon[128791]: pgmap v171: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 9.3 KiB/s rd, 3.4 KiB/s wr, 13 op/s 2023-07-14T11:17:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:47 smithi130 ceph-mon[129405]: pgmap v172: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:47 smithi040 ceph-mon[97954]: pgmap v172: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:48.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:47 smithi045 ceph-mon[128791]: pgmap v172: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:48.672 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:17:48.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.894+0000 7f8009f9d700 1 -- 172.21.15.40:0/3556781968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80041013b0 msgr2=0x7f8004109a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.894+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3556781968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80041013b0 0x7f8004109a80 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7f7ff0009990 tx=0x7f7ff0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 -- 172.21.15.40:0/3556781968 shutdown_connections 2023-07-14T11:17:48.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3556781968 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8004109fc0 0x7f800410c3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3556781968 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004100aa0 0x7f8004100e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3556781968 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f80041013b0 0x7f8004109a80 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 -- 172.21.15.40:0/3556781968 >> 172.21.15.40:0/3556781968 conn(0x7f80040745d0 msgr2=0x7f80040749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 -- 172.21.15.40:0/3556781968 shutdown_connections 2023-07-14T11:17:48.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.895+0000 7f8009f9d700 1 -- 172.21.15.40:0/3556781968 wait complete. 2023-07-14T11:17:48.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.896+0000 7f8009f9d700 1 Processor -- start 2023-07-14T11:17:48.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.896+0000 7f8009f9d700 1 -- start start 2023-07-14T11:17:48.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.896+0000 7f8009f9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8004100aa0 0x7f800419b7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8009f9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8009f9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 0x7f8004195850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8009f9d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8004195ec0 con 0x7f80041013b0 2023-07-14T11:17:48.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8009f9d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8004196030 con 0x7f8004109fc0 2023-07-14T11:17:48.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8009f9d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80041961f0 con 0x7f8004100aa0 2023-07-14T11:17:48.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f800979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 0x7f8004195850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8004100aa0 0x7f800419b7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59258/0 (socket says 172.21.15.40:59258) 2023-07-14T11:17:48.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 -- 172.21.15.40:0/3953217826 learned_addr learned my addr 172.21.15.40:0/3953217826 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:48.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 -- 172.21.15.40:0/3953217826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8004100aa0 msgr2=0x7f800419b7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8004100aa0 0x7f800419b7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 -- 172.21.15.40:0/3953217826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 msgr2=0x7f8004195850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 0x7f8004195850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f8003fff700 1 -- 172.21.15.40:0/3953217826 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7ff00095f0 con 0x7f80041013b0 2023-07-14T11:17:48.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.897+0000 7f800979c700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 0x7f8004195850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:17:48.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8003fff700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f7ff00095c0 tx=0x7f7ff0004730 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8001ffb700 1 -- 172.21.15.40:0/3953217826 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff001d070 con 0x7f80041013b0 2023-07-14T11:17:48.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8001ffb700 1 -- 172.21.15.40:0/3953217826 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ff0022470 con 0x7f80041013b0 2023-07-14T11:17:48.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8004196470 con 0x7f80041013b0 2023-07-14T11:17:48.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8001ffb700 1 -- 172.21.15.40:0/3953217826 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff0017650 con 0x7f80041013b0 2023-07-14T11:17:48.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.898+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80040fe9e0 con 0x7f80041013b0 2023-07-14T11:17:48.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.903+0000 7f8001ffb700 1 -- 172.21.15.40:0/3953217826 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f7ff00225e0 con 0x7f80041013b0 2023-07-14T11:17:48.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.904+0000 7f801ebd7b80 1 Processor -- start 2023-07-14T11:17:48.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.904+0000 7f801ebd7b80 1 -- start start 2023-07-14T11:17:48.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e81d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e827d30 0x55ea7e82a110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e722cb0 con 0x55ea7e827d30 2023-07-14T11:17:48.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e775a40 con 0x55ea7e81d4f0 2023-07-14T11:17:48.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e76eed0 con 0x55ea7e81de00 2023-07-14T11:17:48.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8001ffb700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 0x7f7ff40a3810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49572/0 (socket says 172.21.15.40:49572) 2023-07-14T11:17:48.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 -- 172.21.15.40:0/3371686775 learned_addr learned my addr 172.21.15.40:0/3371686775 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:48.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f800979c700 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e81d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8001ffb700 1 -- 172.21.15.40:0/3953217826 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7ff00ce200 con 0x7f80041013b0 2023-07-14T11:17:48.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8003fff700 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e827d30 0x55ea7e82a110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 -- 172.21.15.40:0/3371686775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 msgr2=0x55ea7e81d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e81d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 -- 172.21.15.40:0/3371686775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e827d30 msgr2=0x55ea7e82a110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e827d30 0x55ea7e82a110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.905+0000 7f8008f9b700 1 -- 172.21.15.40:0/3371686775 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ea7e59cde0 con 0x55ea7e81de00 2023-07-14T11:17:48.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.906+0000 7f800979c700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 0x7f7ff40a3810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.906+0000 7f8008f9b700 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f7ff8009d80 tx=0x7f7ff8011720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f6793b65c8742c71 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.906+0000 7f7feeffd700 1 -- 172.21.15.40:0/3371686775 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff8004230 con 0x55ea7e81de00 2023-07-14T11:17:48.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.906+0000 7f7feeffd700 1 -- 172.21.15.40:0/3371686775 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ff8004390 con 0x55ea7e81de00 2023-07-14T11:17:48.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.906+0000 7f7feeffd700 1 -- 172.21.15.40:0/3371686775 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff8019880 con 0x55ea7e81de00 2023-07-14T11:17:48.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 -- 172.21.15.40:0/3371686775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 msgr2=0x55ea7e8277f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f800979c700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 0x7f7ff40a3810 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7ffc009fd0 tx=0x7f7ffc00b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7f7ff8009d80 tx=0x7f7ff8011720 comp rx=0 tx=0).stop 2023-07-14T11:17:48.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 -- 172.21.15.40:0/3371686775 shutdown_connections 2023-07-14T11:17:48.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e827d30 0x55ea7e82a110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e81d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/3371686775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8277f0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 -- 172.21.15.40:0/3371686775 >> 172.21.15.40:0/3371686775 conn(0x55ea7e812ff0 msgr2=0x55ea7e818930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 -- 172.21.15.40:0/3371686775 shutdown_connections 2023-07-14T11:17:48.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.907+0000 7f801ebd7b80 1 -- 172.21.15.40:0/3371686775 wait complete. 2023-07-14T11:17:48.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.908+0000 7f801ebd7b80 1 -- start start 2023-07-14T11:17:48.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.908+0000 7f801ebd7b80 1 Processor -- start 2023-07-14T11:17:48.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 0x55ea7e824900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e824f10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e70eac0 con 0x55ea7e81de00 2023-07-14T11:17:48.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e70e7c0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e5f2920 con 0x55ea7e827d30 2023-07-14T11:17:48.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f801ebd7b80 10 client.0 _ll_register_callbacks cb 0x55ea7e70c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:17:48.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39300/0 (socket says 172.21.15.40:39300) 2023-07-14T11:17:48.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 -- 172.21.15.40:0/1431746264 learned_addr learned my addr 172.21.15.40:0/1431746264 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:48.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8003fff700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 0x55ea7e824900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f800979c700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e824f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 -- 172.21.15.40:0/1431746264 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 msgr2=0x55ea7e824f10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e824f10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 -- 172.21.15.40:0/1431746264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 msgr2=0x55ea7e824900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 0x55ea7e824900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8008f9b700 1 -- 172.21.15.40:0/1431746264 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7ff80099e0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.909+0000 7f8003fff700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 0x55ea7e824900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f8008f9b700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f7ff801b040 tx=0x7f7ff8003960 comp rx=0 tx=0).ready entity=mon.2 client_cookie=27ccaef13eb090c9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f7fe3fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff8005450 con 0x55ea7e81d4f0 2023-07-14T11:17:48.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ea7e6c84f0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7ff8005ce0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=65) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7ff8009ed0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f801ebd7b80 10 client.26339 fetch_fsmap learned FSMap version 65 2023-07-14T11:17:48.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.910+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=65,monmap=4+}) v3 -- 0x55ea7e585c60 con 0x55ea7e81d4f0 2023-07-14T11:17:48.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ea7e6c84f0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 65) v1 ==== 7498+0+0 (secure 0 0 0) 0x7f7ff8021040 con 0x55ea7e81d4f0 2023-07-14T11:17:48.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 10 client.26339 fetch_fsmap finished waiting for FSMap version 65 2023-07-14T11:17:48.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 10 client.26339 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:17:48.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 20 client.26339 populate_metadata read hostname 'smithi040' 2023-07-14T11:17:48.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e837550 0x55ea7e839960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 4 client.26339 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:17:48.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ea7e767eb0 con 0x55ea7e837550 2023-07-14T11:17:48.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.911+0000 7f8003fff700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e837550 0x55ea7e839960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.912+0000 7f8003fff700 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e837550 0x55ea7e839960 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3880b0f7afc2ccf2 server_cookie=fa1d14202db84596 in_seq=0 out_seq=0 2023-07-14T11:17:48.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.912+0000 7f7fe3fff700 10 client.26339 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:48.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f7fe3fff700 1 -- 172.21.15.40:0/1431746264 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55ea7e767eb0 con 0x55ea7e837550 2023-07-14T11:17:48.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f7fe3fff700 10 client.26339 handle_command_reply: tid=0 2023-07-14T11:17:48.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 client.26339 shutdown 2023-07-14T11:17:48.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 msgr2=0x55ea7e8276f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f7ff801b040 tx=0x7f7ff8003960 comp rx=0 tx=0).stop 2023-07-14T11:17:48.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 shutdown_connections 2023-07-14T11:17:48.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e837550 0x55ea7e839960 crc :-1 s=READY pgs=93 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e824f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81d4f0 0x55ea7e8276f0 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f7fe3fff700 0 client.26339 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:48.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1431746264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81de00 0x55ea7e824900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 >> 172.21.15.40:0/1431746264 conn(0x55ea7e812ff0 msgr2=0x55ea7e818520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 shutdown_connections 2023-07-14T11:17:48.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1431746264 wait complete. 2023-07-14T11:17:48.928 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.913+0000 7f801ebd7b80 20 client.26339 trim_cache size 0 max 16384 2023-07-14T11:17:48.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 Processor -- start 2023-07-14T11:17:48.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 -- start start 2023-07-14T11:17:48.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e827220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e827d30 0x55ea7e8246e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e8194e0 con 0x55ea7e81d4f0 2023-07-14T11:17:48.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e812b70 con 0x55ea7e827d30 2023-07-14T11:17:48.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.931+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e81f1a0 con 0x55ea7e81de00 2023-07-14T11:17:48.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49600/0 (socket says 172.21.15.40:49600) 2023-07-14T11:17:48.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 -- 172.21.15.40:0/1813419238 learned_addr learned my addr 172.21.15.40:0/1813419238 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:48.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8008f9b700 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e827220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f800979c700 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e827d30 0x55ea7e8246e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 -- 172.21.15.40:0/1813419238 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e827d30 msgr2=0x55ea7e8246e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e827d30 0x55ea7e8246e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 -- 172.21.15.40:0/1813419238 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 msgr2=0x55ea7e827220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e827220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 -- 172.21.15.40:0/1813419238 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ea7e5c3250 con 0x55ea7e81de00 2023-07-14T11:17:48.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f8003fff700 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f7ff009af80 tx=0x7f7ff0009ca0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=697379d712bbf18f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.932+0000 7f7fef7fe700 1 -- 172.21.15.40:0/1813419238 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff009a950 con 0x55ea7e81de00 2023-07-14T11:17:48.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f7fef7fe700 1 -- 172.21.15.40:0/1813419238 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ff009aff0 con 0x55ea7e81de00 2023-07-14T11:17:48.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f7fef7fe700 1 -- 172.21.15.40:0/1813419238 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff00cb860 con 0x55ea7e81de00 2023-07-14T11:17:48.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1813419238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 msgr2=0x55ea7e8241a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7f7ff009af80 tx=0x7f7ff0009ca0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1813419238 shutdown_connections 2023-07-14T11:17:48.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e827d30 0x55ea7e8246e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e827220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/1813419238 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e81de00 0x55ea7e8241a0 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1813419238 >> 172.21.15.40:0/1813419238 conn(0x55ea7e812ff0 msgr2=0x55ea7e81fcf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1813419238 shutdown_connections 2023-07-14T11:17:48.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.933+0000 7f801ebd7b80 1 -- 172.21.15.40:0/1813419238 wait complete. 2023-07-14T11:17:48.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.934+0000 7f801ebd7b80 1 -- start start 2023-07-14T11:17:48.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.934+0000 7f801ebd7b80 1 Processor -- start 2023-07-14T11:17:48.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e82d040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e82d750 con 0x55ea7e81d4f0 2023-07-14T11:17:48.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e82d8c0 con 0x55ea7e81de00 2023-07-14T11:17:48.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ea7e88b420 con 0x55ea7e827d30 2023-07-14T11:17:48.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f801ebd7b80 10 client.0 _ll_register_callbacks cb 0x7f7ffc009e60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:17:48.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f800979c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e82d040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:39308/0 (socket says 172.21.15.40:39308) 2023-07-14T11:17:48.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8008f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59302/0 (socket says 172.21.15.40:59302) 2023-07-14T11:17:48.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 -- 172.21.15.40:0/723370589 learned_addr learned my addr 172.21.15.40:0/723370589 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:17:48.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 -- 172.21.15.40:0/723370589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 msgr2=0x55ea7e82d040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e82d040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 -- 172.21.15.40:0/723370589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 msgr2=0x55ea7e82e7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:17:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8003fff700 1 -- 172.21.15.40:0/723370589 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80040fe9e0 con 0x55ea7e81de00 2023-07-14T11:17:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.935+0000 7f8008f9b700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:17:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f8003fff700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f7ff00974f0 tx=0x7f7ff00cb580 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f42a51333bcc9ffa server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:17:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f7fe3fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:17:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7ff006ede0 con 0x55ea7e81de00 2023-07-14T11:17:48.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f7ff00a33b0 con 0x55ea7e81de00 2023-07-14T11:17:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7ff8004390 con 0x55ea7e81de00 2023-07-14T11:17:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=65) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7ff00cb860 con 0x55ea7e81de00 2023-07-14T11:17:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f801ebd7b80 10 client.26345 fetch_fsmap learned FSMap version 65 2023-07-14T11:17:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=65,monmap=4+}) v3 -- 0x55ea7e59b2b0 con 0x55ea7e81de00 2023-07-14T11:17:48.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.936+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7ff8004390 con 0x55ea7e81de00 2023-07-14T11:17:48.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 65) v1 ==== 7498+0+0 (secure 0 0 0) 0x7f7ff009a4e0 con 0x55ea7e81de00 2023-07-14T11:17:48.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 10 client.26345 fetch_fsmap finished waiting for FSMap version 65 2023-07-14T11:17:48.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 10 client.26345 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:17:48.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 20 client.26345 populate_metadata read hostname 'smithi040' 2023-07-14T11:17:48.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e88c590 0x55ea7e88e970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 4 client.26345 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ea7e72a050 con 0x55ea7e88c590 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f8008f9b700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e88c590 0x55ea7e88e970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f8008f9b700 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e88c590 0x55ea7e88e970 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=705d6f564afd2e02 server_cookie=b05652a0a53ec5f6 in_seq=0 out_seq=0 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.937+0000 7f7fe3fff700 10 client.26345 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:48.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f7fe3fff700 1 -- 172.21.15.40:0/723370589 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55ea7e72a050 con 0x55ea7e88c590 2023-07-14T11:17:48.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f7fe3fff700 10 client.26345 handle_command_reply: tid=0 2023-07-14T11:17:48.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 client.26345 shutdown 2023-07-14T11:17:48.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 msgr2=0x55ea7e82cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f7ff00974f0 tx=0x7f7ff00cb580 comp rx=0 tx=0).stop 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 shutdown_connections 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ea7e88c590 0x55ea7e88e970 crc :-1 s=READY pgs=94 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ea7e827d30 0x55ea7e82d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f7fe3fff700 0 client.26345 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ea7e81d4f0 0x55ea7e82e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 --2- 172.21.15.40:0/723370589 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ea7e81de00 0x55ea7e82cb00 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 >> 172.21.15.40:0/723370589 conn(0x55ea7e812ff0 msgr2=0x55ea7e81c440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 shutdown_connections 2023-07-14T11:17:48.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 1 -- 172.21.15.40:0/723370589 wait complete. 2023-07-14T11:17:48.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.938+0000 7f801ebd7b80 20 client.26345 trim_cache size 0 max 16384 2023-07-14T11:17:48.959 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:17:48.959 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:17:48.959 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:17:48.960 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:17:48.960 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:17:48.960 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:17:48.960 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:17:48.960 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:17:48.961 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:17:48.961 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:17:48.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 msgr2=0x7f7ff40a3810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 0x7f7ff40a3810 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f7ffc009fd0 tx=0x7f7ffc00b040 comp rx=0 tx=0).stop 2023-07-14T11:17:48.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 msgr2=0x7f800419bd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:17:48.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 crypto rx=0x7f7ff00095c0 tx=0x7f7ff0004730 comp rx=0 tx=0).stop 2023-07-14T11:17:48.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 shutdown_connections 2023-07-14T11:17:48.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f7ff40a1360 0x7f7ff40a3810 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8004109fc0 0x7f8004195850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8004100aa0 0x7f800419b7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 --2- 172.21.15.40:0/3953217826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f80041013b0 0x7f800419bd20 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:17:48.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 >> 172.21.15.40:0/3953217826 conn(0x7f80040745d0 msgr2=0x7f8004103b60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:17:48.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 shutdown_connections 2023-07-14T11:17:48.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:17:48.939+0000 7f8009f9d700 1 -- 172.21.15.40:0/3953217826 wait complete. 2023-07-14T11:17:48.964 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: 1863 slow requests, 5 included below; oldest blocked for > 227.160054 secs 2023-07-14T11:17:48.964 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: slow request 226.746277 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:17:48.965 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: slow request 226.740320 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:17:48.965 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: slow request 226.734442 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:17:48.965 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: slow request 226.728581 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:17:48.965 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:48 smithi040 ceph-mon[97954]: slow request 226.722704 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:17:48.966 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:17:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: 1863 slow requests, 5 included below; oldest blocked for > 227.160054 secs 2023-07-14T11:17:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: slow request 226.746277 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:17:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: slow request 226.740320 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:17:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: slow request 226.734442 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:17:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: slow request 226.728581 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:17:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:48 smithi130 ceph-mon[129405]: slow request 226.722704 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:17:49.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: 1863 slow requests, 5 included below; oldest blocked for > 227.160054 secs 2023-07-14T11:17:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: slow request 226.746277 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:17:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: slow request 226.740320 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:17:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: slow request 226.734442 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:17:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: slow request 226.728581 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:17:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:48 smithi045 ceph-mon[128791]: slow request 226.722704 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:17:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: 1863 slow requests, 5 included below; oldest blocked for > 230.064662 secs 2023-07-14T11:17:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: slow request 228.338737 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: slow request 228.312338 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: slow request 228.286325 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: slow request 228.260219 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: slow request 228.233575 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:49 smithi130 ceph-mon[129405]: pgmap v173: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: 1863 slow requests, 5 included below; oldest blocked for > 230.064662 secs 2023-07-14T11:17:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: slow request 228.338737 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: slow request 228.312338 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: slow request 228.286325 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: slow request 228.260219 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: slow request 228.233575 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:49 smithi040 ceph-mon[97954]: pgmap v173: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: 1863 slow requests, 5 included below; oldest blocked for > 230.064662 secs 2023-07-14T11:17:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: slow request 228.338737 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: slow request 228.312338 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: slow request 228.286325 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: slow request 228.260219 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: slow request 228.233575 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:49 smithi045 ceph-mon[128791]: pgmap v173: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 1023 B/s wr, 8 op/s 2023-07-14T11:17:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:51 smithi130 ceph-mon[129405]: pgmap v174: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2023-07-14T11:17:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:51 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:51 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:51 smithi040 ceph-mon[97954]: pgmap v174: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2023-07-14T11:17:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:51 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:51 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:52.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:51 smithi045 ceph-mon[128791]: pgmap v174: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 1023 B/s wr, 12 op/s 2023-07-14T11:17:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:51 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:51 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: pgmap v175: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:17:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: 1852 slow requests, 5 included below; oldest blocked for > 232.160118 secs 2023-07-14T11:17:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: slow request 231.716888 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:17:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: slow request 231.710966 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:17:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: slow request 231.705110 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:17:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: slow request 231.699042 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:17:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:53 smithi130 ceph-mon[129405]: slow request 231.693139 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:17:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: pgmap v175: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:17:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: 1852 slow requests, 5 included below; oldest blocked for > 232.160118 secs 2023-07-14T11:17:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: slow request 231.716888 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:17:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: slow request 231.710966 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:17:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: slow request 231.705110 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:17:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: slow request 231.699042 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:17:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:53 smithi040 ceph-mon[97954]: slow request 231.693139 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:17:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: pgmap v175: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:17:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: 1852 slow requests, 5 included below; oldest blocked for > 232.160118 secs 2023-07-14T11:17:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: slow request 231.716888 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:17:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: slow request 231.710966 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:17:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: slow request 231.705110 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:17:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: slow request 231.699042 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:17:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:53 smithi045 ceph-mon[128791]: slow request 231.693139 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:17:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: 1852 slow requests, 5 included below; oldest blocked for > 235.101927 secs 2023-07-14T11:17:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: slow request 233.242334 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: slow request 233.214819 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: slow request 233.187162 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: slow request 233.161500 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:54 smithi130 ceph-mon[129405]: slow request 233.134849 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: 1852 slow requests, 5 included below; oldest blocked for > 235.101927 secs 2023-07-14T11:17:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: slow request 233.242334 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: slow request 233.214819 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: slow request 233.187162 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: slow request 233.161500 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:54 smithi040 ceph-mon[97954]: slow request 233.134849 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: 1852 slow requests, 5 included below; oldest blocked for > 235.101927 secs 2023-07-14T11:17:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: slow request 233.242334 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: slow request 233.214819 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: slow request 233.187162 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: slow request 233.161500 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:54 smithi045 ceph-mon[128791]: slow request 233.134849 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:55 smithi130 ceph-mon[129405]: pgmap v176: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:17:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:55 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:55 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:56.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:55 smithi040 ceph-mon[97954]: pgmap v176: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:17:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:55 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:55 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:55 smithi045 ceph-mon[128791]: pgmap v176: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:17:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:55 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:17:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:55 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:57.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:57 smithi130 ceph-mon[129405]: pgmap v177: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:17:57.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:57 smithi040 ceph-mon[97954]: pgmap v177: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:17:57.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:17:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:17:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:57 smithi045 ceph-mon[128791]: pgmap v177: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:17:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: 1842 slow requests, 5 included below; oldest blocked for > 237.160203 secs 2023-07-14T11:17:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: slow request 236.687459 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:17:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: slow request 236.681517 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:17:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: slow request 236.675659 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:17:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: slow request 236.669762 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:17:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:58 smithi130 ceph-mon[129405]: slow request 236.663867 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:17:58.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: 1842 slow requests, 5 included below; oldest blocked for > 237.160203 secs 2023-07-14T11:17:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: slow request 236.687459 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:17:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: slow request 236.681517 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:17:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: slow request 236.675659 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:17:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: slow request 236.669762 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:17:58.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:58 smithi040 ceph-mon[97954]: slow request 236.663867 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:17:58.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: 1842 slow requests, 5 included below; oldest blocked for > 237.160203 secs 2023-07-14T11:17:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: slow request 236.687459 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:17:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: slow request 236.681517 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:17:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: slow request 236.675659 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:17:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: slow request 236.669762 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:17:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:58 smithi045 ceph-mon[128791]: slow request 236.663867 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:17:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: 1841 slow requests, 5 included below; oldest blocked for > 240.318953 secs 2023-07-14T11:17:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: slow request 238.325122 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: slow request 238.298975 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: slow request 238.272457 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: slow request 238.246433 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: slow request 238.220153 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:17:59 smithi130 ceph-mon[129405]: pgmap v178: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:17:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: 1841 slow requests, 5 included below; oldest blocked for > 240.318953 secs 2023-07-14T11:17:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: slow request 238.325122 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: slow request 238.298975 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: slow request 238.272457 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: slow request 238.246433 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: slow request 238.220153 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:17:59 smithi040 ceph-mon[97954]: pgmap v178: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:17:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: 1841 slow requests, 5 included below; oldest blocked for > 240.318953 secs 2023-07-14T11:17:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: slow request 238.325122 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: slow request 238.298975 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: slow request 238.272457 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: slow request 238.246433 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: slow request 238.220153 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:17:59.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:17:59 smithi045 ceph-mon[128791]: pgmap v178: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 255 B/s wr, 8 op/s 2023-07-14T11:18:01.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:01.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:01.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:01 smithi130 ceph-mon[129405]: pgmap v179: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 938 B/s wr, 13 op/s 2023-07-14T11:18:01.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:01.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:01.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:01 smithi040 ceph-mon[97954]: pgmap v179: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 938 B/s wr, 13 op/s 2023-07-14T11:18:01.813 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:01 smithi045 ceph-mon[128791]: pgmap v179: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.7 KiB/s rd, 938 B/s wr, 13 op/s 2023-07-14T11:18:03.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:03.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:03.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: pgmap v180: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 938 B/s wr, 9 op/s 2023-07-14T11:18:03.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: 1831 slow requests, 5 included below; oldest blocked for > 242.160262 secs 2023-07-14T11:18:03.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: slow request 241.658091 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:18:03.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: slow request 242.066555 seconds old, received at 2023-07-14T11:14:01.027263+0000: peer_request:mds.1:1062 currently dispatched 2023-07-14T11:18:03.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: slow request 242.058247 seconds old, received at 2023-07-14T11:14:01.035572+0000: peer_request:mds.1:1094 currently dispatched 2023-07-14T11:18:03.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: slow request 242.052514 seconds old, received at 2023-07-14T11:14:01.041304+0000: peer_request:mds.1:1126 currently dispatched 2023-07-14T11:18:03.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:03 smithi130 ceph-mon[129405]: slow request 242.042702 seconds old, received at 2023-07-14T11:14:01.051117+0000: peer_request:mds.1:1158 currently dispatched 2023-07-14T11:18:03.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:03.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:03.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: pgmap v180: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 938 B/s wr, 9 op/s 2023-07-14T11:18:03.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: 1831 slow requests, 5 included below; oldest blocked for > 242.160262 secs 2023-07-14T11:18:03.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: slow request 241.658091 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:18:03.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: slow request 242.066555 seconds old, received at 2023-07-14T11:14:01.027263+0000: peer_request:mds.1:1062 currently dispatched 2023-07-14T11:18:03.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: slow request 242.058247 seconds old, received at 2023-07-14T11:14:01.035572+0000: peer_request:mds.1:1094 currently dispatched 2023-07-14T11:18:03.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: slow request 242.052514 seconds old, received at 2023-07-14T11:14:01.041304+0000: peer_request:mds.1:1126 currently dispatched 2023-07-14T11:18:03.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:03 smithi040 ceph-mon[97954]: slow request 242.042702 seconds old, received at 2023-07-14T11:14:01.051117+0000: peer_request:mds.1:1158 currently dispatched 2023-07-14T11:18:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: pgmap v180: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.6 KiB/s rd, 938 B/s wr, 9 op/s 2023-07-14T11:18:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: 1831 slow requests, 5 included below; oldest blocked for > 242.160262 secs 2023-07-14T11:18:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: slow request 241.658091 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:18:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: slow request 242.066555 seconds old, received at 2023-07-14T11:14:01.027263+0000: peer_request:mds.1:1062 currently dispatched 2023-07-14T11:18:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: slow request 242.058247 seconds old, received at 2023-07-14T11:14:01.035572+0000: peer_request:mds.1:1094 currently dispatched 2023-07-14T11:18:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: slow request 242.052514 seconds old, received at 2023-07-14T11:14:01.041304+0000: peer_request:mds.1:1126 currently dispatched 2023-07-14T11:18:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:03 smithi045 ceph-mon[128791]: slow request 242.042702 seconds old, received at 2023-07-14T11:14:01.051117+0000: peer_request:mds.1:1158 currently dispatched 2023-07-14T11:18:04.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: 1830 slow requests, 5 included below; oldest blocked for > 245.532993 secs 2023-07-14T11:18:04.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: slow request 243.406651 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: slow request 245.091955 seconds old, received at 2023-07-14T11:13:58.543618+0000: client_request(mds.1:1077 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: slow request 245.064374 seconds old, received at 2023-07-14T11:13:58.571199+0000: client_request(mds.1:1109 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: slow request 245.032238 seconds old, received at 2023-07-14T11:13:58.603334+0000: client_request(mds.1:1141 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:04 smithi130 ceph-mon[129405]: slow request 244.999693 seconds old, received at 2023-07-14T11:13:58.635880+0000: client_request(mds.1:1173 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: 1830 slow requests, 5 included below; oldest blocked for > 245.532993 secs 2023-07-14T11:18:04.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: slow request 243.406651 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: slow request 245.091955 seconds old, received at 2023-07-14T11:13:58.543618+0000: client_request(mds.1:1077 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: slow request 245.064374 seconds old, received at 2023-07-14T11:13:58.571199+0000: client_request(mds.1:1109 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: slow request 245.032238 seconds old, received at 2023-07-14T11:13:58.603334+0000: client_request(mds.1:1141 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:04 smithi040 ceph-mon[97954]: slow request 244.999693 seconds old, received at 2023-07-14T11:13:58.635880+0000: client_request(mds.1:1173 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: 1830 slow requests, 5 included below; oldest blocked for > 245.532993 secs 2023-07-14T11:18:04.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: slow request 243.406651 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: slow request 245.091955 seconds old, received at 2023-07-14T11:13:58.543618+0000: client_request(mds.1:1077 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: slow request 245.064374 seconds old, received at 2023-07-14T11:13:58.571199+0000: client_request(mds.1:1109 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: slow request 245.032238 seconds old, received at 2023-07-14T11:13:58.603334+0000: client_request(mds.1:1141 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:04.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:04 smithi045 ceph-mon[128791]: slow request 244.999693 seconds old, received at 2023-07-14T11:13:58.635880+0000: client_request(mds.1:1173 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:05.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:05 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:05.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:05 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:05.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:05 smithi130 ceph-mon[129405]: pgmap v181: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 23 KiB/s wr, 26 op/s 2023-07-14T11:18:05.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:05 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:05.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:05 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:05.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:05 smithi040 ceph-mon[97954]: pgmap v181: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 23 KiB/s wr, 26 op/s 2023-07-14T11:18:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:05 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:05 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:05 smithi045 ceph-mon[128791]: pgmap v181: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 23 KiB/s wr, 26 op/s 2023-07-14T11:18:07.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:07.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:07.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:07 smithi130 ceph-mon[129405]: pgmap v182: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:07.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:07.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:07.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:07 smithi040 ceph-mon[97954]: pgmap v182: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:07.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:07.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:07.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:07 smithi045 ceph-mon[128791]: pgmap v182: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: 1821 slow requests, 5 included below; oldest blocked for > 247.160374 secs 2023-07-14T11:18:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: slow request 247.036502 seconds old, received at 2023-07-14T11:14:01.057428+0000: peer_request:mds.1:1190 currently dispatched 2023-07-14T11:18:08.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: slow request 247.029870 seconds old, received at 2023-07-14T11:14:01.064060+0000: peer_request:mds.1:1222 currently dispatched 2023-07-14T11:18:08.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: slow request 247.021108 seconds old, received at 2023-07-14T11:14:01.072822+0000: peer_request:mds.1:1254 currently dispatched 2023-07-14T11:18:08.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: slow request 247.015400 seconds old, received at 2023-07-14T11:14:01.078530+0000: peer_request:mds.1:1286 currently dispatched 2023-07-14T11:18:08.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:08 smithi130 ceph-mon[129405]: slow request 247.007320 seconds old, received at 2023-07-14T11:14:01.086610+0000: peer_request:mds.1:1318 currently dispatched 2023-07-14T11:18:08.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: 1821 slow requests, 5 included below; oldest blocked for > 247.160374 secs 2023-07-14T11:18:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: slow request 247.036502 seconds old, received at 2023-07-14T11:14:01.057428+0000: peer_request:mds.1:1190 currently dispatched 2023-07-14T11:18:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: slow request 247.029870 seconds old, received at 2023-07-14T11:14:01.064060+0000: peer_request:mds.1:1222 currently dispatched 2023-07-14T11:18:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: slow request 247.021108 seconds old, received at 2023-07-14T11:14:01.072822+0000: peer_request:mds.1:1254 currently dispatched 2023-07-14T11:18:08.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: slow request 247.015400 seconds old, received at 2023-07-14T11:14:01.078530+0000: peer_request:mds.1:1286 currently dispatched 2023-07-14T11:18:08.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:08 smithi040 ceph-mon[97954]: slow request 247.007320 seconds old, received at 2023-07-14T11:14:01.086610+0000: peer_request:mds.1:1318 currently dispatched 2023-07-14T11:18:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: 1821 slow requests, 5 included below; oldest blocked for > 247.160374 secs 2023-07-14T11:18:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: slow request 247.036502 seconds old, received at 2023-07-14T11:14:01.057428+0000: peer_request:mds.1:1190 currently dispatched 2023-07-14T11:18:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: slow request 247.029870 seconds old, received at 2023-07-14T11:14:01.064060+0000: peer_request:mds.1:1222 currently dispatched 2023-07-14T11:18:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: slow request 247.021108 seconds old, received at 2023-07-14T11:14:01.072822+0000: peer_request:mds.1:1254 currently dispatched 2023-07-14T11:18:08.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: slow request 247.015400 seconds old, received at 2023-07-14T11:14:01.078530+0000: peer_request:mds.1:1286 currently dispatched 2023-07-14T11:18:08.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:08 smithi045 ceph-mon[128791]: slow request 247.007320 seconds old, received at 2023-07-14T11:14:01.086610+0000: peer_request:mds.1:1318 currently dispatched 2023-07-14T11:18:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: pgmap v183: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: 1820 slow requests, 5 included below; oldest blocked for > 250.670022 secs 2023-07-14T11:18:09.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: slow request 250.104655 seconds old, received at 2023-07-14T11:13:58.667947+0000: client_request(mds.1:1205 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: slow request 250.075553 seconds old, received at 2023-07-14T11:13:58.697049+0000: client_request(mds.1:1237 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: slow request 250.035839 seconds old, received at 2023-07-14T11:13:58.736763+0000: client_request(mds.1:1269 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: slow request 250.004057 seconds old, received at 2023-07-14T11:13:58.768544+0000: client_request(mds.1:1301 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:09.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:09 smithi045 ceph-mon[128791]: slow request 249.968468 seconds old, received at 2023-07-14T11:13:58.804134+0000: client_request(mds.1:1333 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: pgmap v183: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: 1820 slow requests, 5 included below; oldest blocked for > 250.670022 secs 2023-07-14T11:18:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: slow request 250.104655 seconds old, received at 2023-07-14T11:13:58.667947+0000: client_request(mds.1:1205 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: slow request 250.075553 seconds old, received at 2023-07-14T11:13:58.697049+0000: client_request(mds.1:1237 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: slow request 250.035839 seconds old, received at 2023-07-14T11:13:58.736763+0000: client_request(mds.1:1269 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: slow request 250.004057 seconds old, received at 2023-07-14T11:13:58.768544+0000: client_request(mds.1:1301 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:09 smithi130 ceph-mon[129405]: slow request 249.968468 seconds old, received at 2023-07-14T11:13:58.804134+0000: client_request(mds.1:1333 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: pgmap v183: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 12 KiB/s rd, 23 KiB/s wr, 21 op/s 2023-07-14T11:18:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: 1820 slow requests, 5 included below; oldest blocked for > 250.670022 secs 2023-07-14T11:18:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: slow request 250.104655 seconds old, received at 2023-07-14T11:13:58.667947+0000: client_request(mds.1:1205 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: slow request 250.075553 seconds old, received at 2023-07-14T11:13:58.697049+0000: client_request(mds.1:1237 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: slow request 250.035839 seconds old, received at 2023-07-14T11:13:58.736763+0000: client_request(mds.1:1269 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: slow request 250.004057 seconds old, received at 2023-07-14T11:13:58.768544+0000: client_request(mds.1:1301 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:10.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:09 smithi040 ceph-mon[97954]: slow request 249.968468 seconds old, received at 2023-07-14T11:13:58.804134+0000: client_request(mds.1:1333 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:11.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:11.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:11 smithi045 ceph-mon[128791]: pgmap v184: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:11 smithi130 ceph-mon[129405]: pgmap v184: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:12.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:11 smithi040 ceph-mon[97954]: pgmap v184: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:13.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:13.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:13.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: pgmap v185: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 24 KiB/s wr, 21 op/s 2023-07-14T11:18:13.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: 1811 slow requests, 5 included below; oldest blocked for > 252.160455 secs 2023-07-14T11:18:13.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: slow request 251.997343 seconds old, received at 2023-07-14T11:14:01.096667+0000: peer_request:mds.1:1350 currently dispatched 2023-07-14T11:18:13.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: slow request 251.987323 seconds old, received at 2023-07-14T11:14:01.106687+0000: peer_request:mds.1:1382 currently dispatched 2023-07-14T11:18:13.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: slow request 251.977230 seconds old, received at 2023-07-14T11:14:01.116780+0000: peer_request:mds.1:1414 currently dispatched 2023-07-14T11:18:13.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: slow request 251.967240 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:18:13.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:13 smithi045 ceph-mon[128791]: slow request 251.957256 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:18:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: pgmap v185: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 24 KiB/s wr, 21 op/s 2023-07-14T11:18:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: 1811 slow requests, 5 included below; oldest blocked for > 252.160455 secs 2023-07-14T11:18:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: slow request 251.997343 seconds old, received at 2023-07-14T11:14:01.096667+0000: peer_request:mds.1:1350 currently dispatched 2023-07-14T11:18:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: slow request 251.987323 seconds old, received at 2023-07-14T11:14:01.106687+0000: peer_request:mds.1:1382 currently dispatched 2023-07-14T11:18:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: slow request 251.977230 seconds old, received at 2023-07-14T11:14:01.116780+0000: peer_request:mds.1:1414 currently dispatched 2023-07-14T11:18:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: slow request 251.967240 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:18:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:13 smithi130 ceph-mon[129405]: slow request 251.957256 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:18:14.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: pgmap v185: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 14 KiB/s rd, 24 KiB/s wr, 21 op/s 2023-07-14T11:18:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: 1811 slow requests, 5 included below; oldest blocked for > 252.160455 secs 2023-07-14T11:18:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: slow request 251.997343 seconds old, received at 2023-07-14T11:14:01.096667+0000: peer_request:mds.1:1350 currently dispatched 2023-07-14T11:18:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: slow request 251.987323 seconds old, received at 2023-07-14T11:14:01.106687+0000: peer_request:mds.1:1382 currently dispatched 2023-07-14T11:18:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: slow request 251.977230 seconds old, received at 2023-07-14T11:14:01.116780+0000: peer_request:mds.1:1414 currently dispatched 2023-07-14T11:18:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: slow request 251.967240 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:18:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:13 smithi040 ceph-mon[97954]: slow request 251.957256 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:18:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: 1809 slow requests, 5 included below; oldest blocked for > 255.827020 secs 2023-07-14T11:18:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: slow request 255.091371 seconds old, received at 2023-07-14T11:13:58.838229+0000: client_request(mds.1:1365 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: slow request 255.060767 seconds old, received at 2023-07-14T11:13:58.868833+0000: client_request(mds.1:1397 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:14.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: slow request 255.031348 seconds old, received at 2023-07-14T11:13:58.898252+0000: client_request(mds.1:1429 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:14.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: slow request 254.996806 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:14.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:14 smithi045 ceph-mon[128791]: slow request 254.955107 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: 1809 slow requests, 5 included below; oldest blocked for > 255.827020 secs 2023-07-14T11:18:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: slow request 255.091371 seconds old, received at 2023-07-14T11:13:58.838229+0000: client_request(mds.1:1365 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: slow request 255.060767 seconds old, received at 2023-07-14T11:13:58.868833+0000: client_request(mds.1:1397 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: slow request 255.031348 seconds old, received at 2023-07-14T11:13:58.898252+0000: client_request(mds.1:1429 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: slow request 254.996806 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:14 smithi130 ceph-mon[129405]: slow request 254.955107 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: 1809 slow requests, 5 included below; oldest blocked for > 255.827020 secs 2023-07-14T11:18:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: slow request 255.091371 seconds old, received at 2023-07-14T11:13:58.838229+0000: client_request(mds.1:1365 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: slow request 255.060767 seconds old, received at 2023-07-14T11:13:58.868833+0000: client_request(mds.1:1397 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: slow request 255.031348 seconds old, received at 2023-07-14T11:13:58.898252+0000: client_request(mds.1:1429 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: slow request 254.996806 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:14 smithi040 ceph-mon[97954]: slow request 254.955107 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:16.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:15 smithi130 ceph-mon[129405]: pgmap v186: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:16.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:15 smithi040 ceph-mon[97954]: pgmap v186: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:15 smithi045 ceph-mon[128791]: pgmap v186: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 25 KiB/s wr, 26 op/s 2023-07-14T11:18:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:17 smithi130 ceph-mon[129405]: pgmap v187: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:18.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:17 smithi040 ceph-mon[97954]: pgmap v187: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:18.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:17 smithi045 ceph-mon[128791]: pgmap v187: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:18.967 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:18:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: 1799 slow requests, 5 included below; oldest blocked for > 257.160570 secs 2023-07-14T11:18:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: slow request 256.947296 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:18:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: slow request 256.937218 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:18:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: slow request 256.927209 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:18:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: slow request 256.919089 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:18:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:18 smithi130 ceph-mon[129405]: slow request 256.909280 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:18:19.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: 1799 slow requests, 5 included below; oldest blocked for > 257.160570 secs 2023-07-14T11:18:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: slow request 256.947296 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:18:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: slow request 256.937218 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:18:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: slow request 256.927209 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:18:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: slow request 256.919089 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:18:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:18 smithi040 ceph-mon[97954]: slow request 256.909280 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:18:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: 1799 slow requests, 5 included below; oldest blocked for > 257.160570 secs 2023-07-14T11:18:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: slow request 256.947296 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:18:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: slow request 256.937218 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:18:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: slow request 256.927209 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:18:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: slow request 256.919089 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:18:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:18 smithi045 ceph-mon[128791]: slow request 256.909280 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:18:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: pgmap v188: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: 1798 slow requests, 5 included below; oldest blocked for > 260.833123 secs 2023-07-14T11:18:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: slow request 259.925073 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: slow request 259.891482 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: slow request 259.864452 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: slow request 259.831057 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:19 smithi130 ceph-mon[129405]: slow request 259.801911 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: pgmap v188: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: 1798 slow requests, 5 included below; oldest blocked for > 260.833123 secs 2023-07-14T11:18:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: slow request 259.925073 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: slow request 259.891482 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: slow request 259.864452 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: slow request 259.831057 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:19 smithi040 ceph-mon[97954]: slow request 259.801911 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: pgmap v188: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.0 KiB/s rd, 2.2 KiB/s wr, 9 op/s 2023-07-14T11:18:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: 1798 slow requests, 5 included below; oldest blocked for > 260.833123 secs 2023-07-14T11:18:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: slow request 259.925073 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: slow request 259.891482 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: slow request 259.864452 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: slow request 259.831057 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:20.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:19 smithi045 ceph-mon[128791]: slow request 259.801911 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:21 smithi130 ceph-mon[129405]: pgmap v189: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 8.1 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2023-07-14T11:18:22.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:21 smithi040 ceph-mon[97954]: pgmap v189: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 8.1 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2023-07-14T11:18:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:21 smithi045 ceph-mon[128791]: pgmap v189: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 8.1 KiB/s rd, 2.2 KiB/s wr, 13 op/s 2023-07-14T11:18:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: pgmap v190: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 170 B/s wr, 8 op/s 2023-07-14T11:18:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: 1789 slow requests, 5 included below; oldest blocked for > 262.160676 secs 2023-07-14T11:18:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: slow request 261.899498 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:18:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: slow request 261.889492 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:18:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: slow request 261.879455 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:18:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: slow request 261.869355 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:18:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: slow request 261.859299 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:18:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:23 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:18:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: pgmap v190: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 170 B/s wr, 8 op/s 2023-07-14T11:18:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: 1789 slow requests, 5 included below; oldest blocked for > 262.160676 secs 2023-07-14T11:18:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: slow request 261.899498 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:18:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: slow request 261.889492 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:18:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: slow request 261.879455 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:18:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: slow request 261.869355 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:18:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: slow request 261.859299 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:18:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:23 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:18:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: pgmap v190: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.3 KiB/s rd, 170 B/s wr, 8 op/s 2023-07-14T11:18:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: 1789 slow requests, 5 included below; oldest blocked for > 262.160676 secs 2023-07-14T11:18:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: slow request 261.899498 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:18:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: slow request 261.889492 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:18:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: slow request 261.879455 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:18:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: slow request 261.869355 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:18:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: slow request 261.859299 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:18:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:23 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:18:24.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/3103186923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d580fe820 msgr2=0x7f0d580febf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/3103186923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d580fe820 0x7f0d580febf0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f0d48009990 tx=0x7f0d48009ca0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/3103186923 shutdown_connections 2023-07-14T11:18:24.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/3103186923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d5810c3d0 0x7f0d5810e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/3103186923 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d580fe820 0x7f0d580febf0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/3103186923 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580ff1c0 0x7f0d5810be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/3103186923 >> 172.21.15.40:0/3103186923 conn(0x7f0d580fa4b0 msgr2=0x7f0d580fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/3103186923 shutdown_connections 2023-07-14T11:18:24.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.797+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/3103186923 wait complete. 2023-07-14T11:18:24.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.798+0000 7f0d5e3ff700 1 Processor -- start 2023-07-14T11:18:24.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.798+0000 7f0d5e3ff700 1 -- start start 2023-07-14T11:18:24.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580fe820 0x7f0d58197e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d580ff1c0 0x7f0d58198360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d5819ce00 con 0x7f0d580ff1c0 2023-07-14T11:18:24.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d5819cf70 con 0x7f0d5810c3d0 2023-07-14T11:18:24.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5e3ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0d5819d130 con 0x7f0d580fe820 2023-07-14T11:18:24.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5c99c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5c99c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:42724/0 (socket says 172.21.15.40:42724) 2023-07-14T11:18:24.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d5c99c700 1 -- 172.21.15.40:0/1112424022 learned_addr learned my addr 172.21.15.40:0/1112424022 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:24.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d57fff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580fe820 0x7f0d58197e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.799+0000 7f0d577fe700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d580ff1c0 0x7f0d58198360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 -- 172.21.15.40:0/1112424022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580fe820 msgr2=0x7f0d58197e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580fe820 0x7f0d58197e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 -- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d580ff1c0 msgr2=0x7f0d58198360 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d580ff1c0 0x7f0d58198360 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 -- 172.21.15.40:0/1112424022 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0d480095f0 con 0x7f0d5810c3d0 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f0d50009fd0 tx=0x7f0d5000ef90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d557fa700 1 -- 172.21.15.40:0/1112424022 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d50018070 con 0x7f0d5810c3d0 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d557fa700 1 -- 172.21.15.40:0/1112424022 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d5000f040 con 0x7f0d5810c3d0 2023-07-14T11:18:24.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d557fa700 1 -- 172.21.15.40:0/1112424022 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d5000cd00 con 0x7f0d5810c3d0 2023-07-14T11:18:24.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.800+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0d5819d3b0 con 0x7f0d5810c3d0 2023-07-14T11:18:24.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.801+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d581a4c50 con 0x7f0d5810c3d0 2023-07-14T11:18:24.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.804+0000 7f0d557fa700 1 -- 172.21.15.40:0/1112424022 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f0d50004ad0 con 0x7f0d5810c3d0 2023-07-14T11:18:24.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.805+0000 7f0d557fa700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 0x7f0d400a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.805+0000 7f0d557fa700 1 -- 172.21.15.40:0/1112424022 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0d500cca20 con 0x7f0d5810c3d0 2023-07-14T11:18:24.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.806+0000 7f0d57fff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 0x7f0d400a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.806+0000 7f0d57fff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 0x7f0d400a3770 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f0d4800b2e0 tx=0x7f0d48005fd0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.819+0000 7f0d71d4bb80 1 Processor -- start 2023-07-14T11:18:24.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 -- start start 2023-07-14T11:18:24.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f02590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f04eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682dfdcb0 con 0x559682f02ad0 2023-07-14T11:18:24.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682e50a40 con 0x559682ef7be0 2023-07-14T11:18:24.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682e49ed0 con 0x559682ef72d0 2023-07-14T11:18:24.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f02590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:38932/0 (socket says 172.21.15.40:38932) 2023-07-14T11:18:24.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.820+0000 7f0d57fff700 1 -- 172.21.15.40:0/3925068775 learned_addr learned my addr 172.21.15.40:0/3925068775 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:24.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f04eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 -- 172.21.15.40:0/3925068775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 msgr2=0x559682f02590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f02590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 -- 172.21.15.40:0/3925068775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 msgr2=0x559682f04eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f04eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 -- 172.21.15.40:0/3925068775 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559682c77de0 con 0x559682ef72d0 2023-07-14T11:18:24.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.821+0000 7f0d57fff700 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0d48015000 tx=0x7f0d48020000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bb0dc0949b88b722 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d3e7fc700 1 -- 172.21.15.40:0/3925068775 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d4802b070 con 0x559682ef72d0 2023-07-14T11:18:24.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d3e7fc700 1 -- 172.21.15.40:0/3925068775 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d4801e040 con 0x559682ef72d0 2023-07-14T11:18:24.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d3e7fc700 1 -- 172.21.15.40:0/3925068775 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d4800f660 con 0x559682ef72d0 2023-07-14T11:18:24.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/3925068775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 msgr2=0x559682ef76a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f0d48015000 tx=0x7f0d48020000 comp rx=0 tx=0).stop 2023-07-14T11:18:24.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/3925068775 shutdown_connections 2023-07-14T11:18:24.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f04eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682ef76a0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.822+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/3925068775 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f02590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.823+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/3925068775 >> 172.21.15.40:0/3925068775 conn(0x559682eedda0 msgr2=0x559682ef36a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.823+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/3925068775 shutdown_connections 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.823+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/3925068775 wait complete. 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.823+0000 7f0d71d4bb80 1 -- start start 2023-07-14T11:18:24.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.823+0000 7f0d71d4bb80 1 Processor -- start 2023-07-14T11:18:24.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f024c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682effe30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682de9ac0 con 0x559682f02ad0 2023-07-14T11:18:24.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682de97c0 con 0x559682ef7be0 2023-07-14T11:18:24.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682ccd920 con 0x559682ef72d0 2023-07-14T11:18:24.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d71d4bb80 10 client.0 _ll_register_callbacks cb 0x559682de7760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:18:24.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:42736/0 (socket says 172.21.15.40:42736) 2023-07-14T11:18:24.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f024c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.824+0000 7f0d577fe700 1 -- 172.21.15.40:0/2227927422 learned_addr learned my addr 172.21.15.40:0/2227927422 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:24.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682effe30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 -- 172.21.15.40:0/2227927422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 msgr2=0x559682f024c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f024c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 -- 172.21.15.40:0/2227927422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 msgr2=0x559682effe30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682effe30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 -- 172.21.15.40:0/2227927422 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0d5810f3f0 con 0x559682ef7be0 2023-07-14T11:18:24.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682effe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:18:24.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d577fe700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f0d4c012770 tx=0x7f0d4c012a80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8491c8e5ebf3d65d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d377fe700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d4c017070 con 0x559682ef7be0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559682da34f0 con 0x559682ef7be0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.825+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0d4c00f4e0 con 0x559682ef7be0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=80) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d4c00f6a0 con 0x559682ef7be0 2023-07-14T11:18:24.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 10 client.26357 fetch_fsmap learned FSMap version 80 2023-07-14T11:18:24.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=80,monmap=4+}) v3 -- 0x559682c60c60 con 0x559682ef7be0 2023-07-14T11:18:24.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559682da34f0 con 0x559682ef7be0 2023-07-14T11:18:24.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 80) v1 ==== 7498+0+0 (secure 0 0 0) 0x7f0d4c0108a0 con 0x559682ef7be0 2023-07-14T11:18:24.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 10 client.26357 fetch_fsmap finished waiting for FSMap version 80 2023-07-14T11:18:24.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 10 client.26357 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:18:24.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 20 client.26357 populate_metadata read hostname 'smithi040' 2023-07-14T11:18:24.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f122f0 0x559682f14700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 4 client.26357 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:18:24.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.826+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559682e42eb0 con 0x559682f122f0 2023-07-14T11:18:24.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.827+0000 7f0d57fff700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f122f0 0x559682f14700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.827+0000 7f0d57fff700 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f122f0 0x559682f14700 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=41436b685d748d9a server_cookie=187daf5f6512eaf in_seq=0 out_seq=0 2023-07-14T11:18:24.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.827+0000 7f0d377fe700 10 client.26357 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:24.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.828+0000 7f0d377fe700 1 -- 172.21.15.40:0/2227927422 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x559682e42eb0 con 0x559682f122f0 2023-07-14T11:18:24.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.828+0000 7f0d377fe700 10 client.26357 handle_command_reply: tid=0 2023-07-14T11:18:24.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.828+0000 7f0d71d4bb80 1 client.26357 shutdown 2023-07-14T11:18:24.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 msgr2=0x559682eff7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f0d4c012770 tx=0x7f0d4c012a80 comp rx=0 tx=0).stop 2023-07-14T11:18:24.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 shutdown_connections 2023-07-14T11:18:24.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f122f0 0x559682f14700 crc :-1 s=READY pgs=95 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682effe30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f024c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d377fe700 0 client.26357 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:24.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2227927422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682eff7b0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 >> 172.21.15.40:0/2227927422 conn(0x559682eedda0 msgr2=0x559682ef3230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 shutdown_connections 2023-07-14T11:18:24.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2227927422 wait complete. 2023-07-14T11:18:24.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.829+0000 7f0d71d4bb80 20 client.26357 trim_cache size 0 max 16384 2023-07-14T11:18:24.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 Processor -- start 2023-07-14T11:18:24.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 -- start start 2023-07-14T11:18:24.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 0x559682f02060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682eeb010 con 0x559682ef72d0 2023-07-14T11:18:24.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682eed9a0 con 0x559682f02ad0 2023-07-14T11:18:24.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.847+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682ef4500 con 0x559682ef7be0 2023-07-14T11:18:24.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 0x559682f02060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:42742/0 (socket says 172.21.15.40:42742) 2023-07-14T11:18:24.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:38946/0 (socket says 172.21.15.40:38946) 2023-07-14T11:18:24.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 -- 172.21.15.40:0/2086079656 learned_addr learned my addr 172.21.15.40:0/2086079656 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:24.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 -- 172.21.15.40:0/2086079656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 msgr2=0x559682efef70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 -- 172.21.15.40:0/2086079656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 msgr2=0x559682f02060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 0x559682f02060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 -- 172.21.15.40:0/2086079656 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559682c9e250 con 0x559682f02ad0 2023-07-14T11:18:24.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d577fe700 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:18:24.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d57fff700 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 0x559682f02060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:18:24.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.848+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f0d5000c680 tx=0x7f0d50099170 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8ac3ba49c4e9cfa0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d3effd700 1 -- 172.21.15.40:0/2086079656 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d50099810 con 0x559682f02ad0 2023-07-14T11:18:24.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d3effd700 1 -- 172.21.15.40:0/2086079656 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d5009d840 con 0x559682f02ad0 2023-07-14T11:18:24.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d3effd700 1 -- 172.21.15.40:0/2086079656 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d50098d00 con 0x559682f02ad0 2023-07-14T11:18:24.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2086079656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 msgr2=0x559682eff4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f0d5000c680 tx=0x7f0d50099170 comp rx=0 tx=0).stop 2023-07-14T11:18:24.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2086079656 shutdown_connections 2023-07-14T11:18:24.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682f02ad0 0x559682eff4b0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682ef72d0 0x559682f02060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/2086079656 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef7be0 0x559682efef70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.849+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2086079656 >> 172.21.15.40:0/2086079656 conn(0x559682eedda0 msgr2=0x559682f0e6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.850+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2086079656 shutdown_connections 2023-07-14T11:18:24.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.850+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/2086079656 wait complete. 2023-07-14T11:18:24.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.850+0000 7f0d71d4bb80 1 -- start start 2023-07-14T11:18:24.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.850+0000 7f0d71d4bb80 1 Processor -- start 2023-07-14T11:18:24.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f081c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f08700 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682f66250 con 0x559682f02ad0 2023-07-14T11:18:24.861 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682f66390 con 0x559682ef7be0 2023-07-14T11:18:24.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559682f66550 con 0x559682ef72d0 2023-07-14T11:18:24.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d71d4bb80 10 client.0 _ll_register_callbacks cb 0x7f0d4c009d90 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:18:24.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d577fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f081c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:38958/0 (socket says 172.21.15.40:38958) 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 -- 172.21.15.40:0/707742138 learned_addr learned my addr 172.21.15.40:0/707742138 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f08700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 -- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 msgr2=0x559682f081c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f081c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 -- 172.21.15.40:0/707742138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 msgr2=0x559682f08700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:24.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.851+0000 7f0d57fff700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f08700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d57fff700 1 -- 172.21.15.40:0/707742138 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x559682c77de0 con 0x559682ef72d0 2023-07-14T11:18:24.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d5c99c700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f08700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:18:24.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d577fe700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:18:24.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d57fff700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f0d48004c40 tx=0x7f0d48004d10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=25f20b71d3d30d87 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:24.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d377fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:18:24.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0d4802b040 con 0x559682ef72d0 2023-07-14T11:18:24.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f0d4801e040 con 0x559682ef72d0 2023-07-14T11:18:24.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.852+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f0d4803cad0 con 0x559682ef72d0 2023-07-14T11:18:24.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=80) v2 ==== 24+0+0 (secure 0 0 0) 0x7f0d4803cc90 con 0x559682ef72d0 2023-07-14T11:18:24.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 10 client.26326 fetch_fsmap learned FSMap version 80 2023-07-14T11:18:24.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=80,monmap=4+}) v3 -- 0x559682c60c60 con 0x559682ef72d0 2023-07-14T11:18:24.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0d4801e040 con 0x559682ef72d0 2023-07-14T11:18:24.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 80) v1 ==== 7498+0+0 (secure 0 0 0) 0x7f0d48005040 con 0x559682ef72d0 2023-07-14T11:18:24.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 10 client.26326 fetch_fsmap finished waiting for FSMap version 80 2023-07-14T11:18:24.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 10 client.26326 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:18:24.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 20 client.26326 populate_metadata read hostname 'smithi040' 2023-07-14T11:18:24.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f677c0 0x559682f69ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:24.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 4 client.26326 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:18:24.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.853+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x559682e05050 con 0x559682f677c0 2023-07-14T11:18:24.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d577fe700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f677c0 0x559682f69ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d377fe700 10 client.26326 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d577fe700 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f677c0 0x559682f69ba0 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b432cae21484180 server_cookie=9a47f905ed3e7788 in_seq=0 out_seq=0 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d377fe700 1 -- 172.21.15.40:0/707742138 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x559682e05050 con 0x559682f677c0 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d377fe700 10 client.26326 handle_command_reply: tid=0 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.854+0000 7f0d71d4bb80 1 client.26326 shutdown 2023-07-14T11:18:24.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 msgr2=0x559682f07c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f0d48004c40 tx=0x7f0d48004d10 comp rx=0 tx=0).stop 2023-07-14T11:18:24.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 shutdown_connections 2023-07-14T11:18:24.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559682f677c0 0x559682f69ba0 crc :-1 s=READY pgs=96 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559682f02ad0 0x559682f08700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559682ef72d0 0x559682f07c80 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d377fe700 0 client.26326 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:24.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 --2- 172.21.15.40:0/707742138 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559682ef7be0 0x559682f081c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 >> 172.21.15.40:0/707742138 conn(0x559682eedda0 msgr2=0x559682ef9260 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 shutdown_connections 2023-07-14T11:18:24.870 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 1 -- 172.21.15.40:0/707742138 wait complete. 2023-07-14T11:18:24.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.855+0000 7f0d71d4bb80 20 client.26326 trim_cache size 0 max 16384 2023-07-14T11:18:24.871 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:18:24.871 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:18:24.871 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:18:24.872 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:18:24.872 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:18:24.872 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:18:24.872 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:18:24.872 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:18:24.873 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:18:24.873 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:18:24.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 msgr2=0x7f0d400a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 0x7f0d400a3770 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f0d4800b2e0 tx=0x7f0d48005fd0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 msgr2=0x7f0d5819c7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:24.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f0d50009fd0 tx=0x7f0d5000ef90 comp rx=0 tx=0).stop 2023-07-14T11:18:24.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 shutdown_connections 2023-07-14T11:18:24.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0d400a12c0 0x7f0d400a3770 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0d5810c3d0 0x7f0d5819c7c0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0d580fe820 0x7f0d58197e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 --2- 172.21.15.40:0/1112424022 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0d580ff1c0 0x7f0d58198360 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:24.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 >> 172.21.15.40:0/1112424022 conn(0x7f0d580fa4b0 msgr2=0x7f0d5810d6e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:24.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 shutdown_connections 2023-07-14T11:18:24.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:24.856+0000 7f0d5e3ff700 1 -- 172.21.15.40:0/1112424022 wait complete. 2023-07-14T11:18:24.885 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:18:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: 1787 slow requests, 5 included below; oldest blocked for > 265.986172 secs 2023-07-14T11:18:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: slow request 264.921913 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: slow request 264.870342 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: slow request 264.813448 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: slow request 264.787068 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: slow request 264.758818 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: pgmap v191: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 170 B/s wr, 12 op/s 2023-07-14T11:18:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:18:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:18:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:25 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:18:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: 1787 slow requests, 5 included below; oldest blocked for > 265.986172 secs 2023-07-14T11:18:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: slow request 264.921913 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: slow request 264.870342 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: slow request 264.813448 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: slow request 264.787068 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: slow request 264.758818 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: pgmap v191: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 170 B/s wr, 12 op/s 2023-07-14T11:18:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:18:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:18:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:25 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:18:26.086 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: 1787 slow requests, 5 included below; oldest blocked for > 265.986172 secs 2023-07-14T11:18:26.086 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: slow request 264.921913 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.086 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: slow request 264.870342 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.086 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: slow request 264.813448 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.087 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: slow request 264.787068 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.087 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: slow request 264.758818 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:26.087 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:26.087 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:26.088 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: pgmap v191: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 170 B/s wr, 12 op/s 2023-07-14T11:18:26.088 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:18:26.088 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:18:26.088 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:25 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:18:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:27 smithi130 ceph-mon[129405]: pgmap v192: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:27 smithi040 ceph-mon[97954]: pgmap v192: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:27 smithi045 ceph-mon[128791]: pgmap v192: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: 1778 slow requests, 5 included below; oldest blocked for > 267.160739 secs 2023-07-14T11:18:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: slow request 266.849362 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:18:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: slow request 266.839442 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:18:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: slow request 266.833710 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:18:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: slow request 266.827942 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:18:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:28 smithi130 ceph-mon[129405]: slow request 266.822256 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:18:29.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: 1778 slow requests, 5 included below; oldest blocked for > 267.160739 secs 2023-07-14T11:18:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: slow request 266.849362 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:18:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: slow request 266.839442 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:18:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: slow request 266.833710 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:18:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: slow request 266.827942 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:18:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:28 smithi040 ceph-mon[97954]: slow request 266.822256 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:18:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: 1778 slow requests, 5 included below; oldest blocked for > 267.160739 secs 2023-07-14T11:18:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: slow request 266.849362 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:18:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: slow request 266.839442 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:18:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: slow request 266.833710 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:18:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: slow request 266.827942 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:18:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:28 smithi045 ceph-mon[128791]: slow request 266.822256 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:18:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:29 smithi130 ceph-mon[129405]: pgmap v193: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:29 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:29 smithi040 ceph-mon[97954]: pgmap v193: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:29 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:30.063 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:29 smithi045 ceph-mon[128791]: pgmap v193: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:29 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: 1776 slow requests, 5 included below; oldest blocked for > 271.106763 secs 2023-07-14T11:18:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: slow request 269.851818 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: slow request 269.825254 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: slow request 269.799015 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: slow request 269.772457 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:30 smithi130 ceph-mon[129405]: slow request 269.746329 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: 1776 slow requests, 5 included below; oldest blocked for > 271.106763 secs 2023-07-14T11:18:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: slow request 269.851818 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: slow request 269.825254 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: slow request 269.799015 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: slow request 269.772457 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:30 smithi040 ceph-mon[97954]: slow request 269.746329 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: 1776 slow requests, 5 included below; oldest blocked for > 271.106763 secs 2023-07-14T11:18:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: slow request 269.851818 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: slow request 269.825254 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: slow request 269.799015 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: slow request 269.772457 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:30 smithi045 ceph-mon[128791]: slow request 269.746329 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:31 smithi130 ceph-mon[129405]: pgmap v194: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:32.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:31 smithi040 ceph-mon[97954]: pgmap v194: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:31 smithi045 ceph-mon[128791]: pgmap v194: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: pgmap v195: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: 1767 slow requests, 5 included below; oldest blocked for > 272.160839 secs 2023-07-14T11:18:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: slow request 271.816522 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:18:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: slow request 271.810665 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:18:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: slow request 271.805055 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:18:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: slow request 271.799345 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:18:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:33 smithi130 ceph-mon[129405]: slow request 271.793603 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:18:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: pgmap v195: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: 1767 slow requests, 5 included below; oldest blocked for > 272.160839 secs 2023-07-14T11:18:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: slow request 271.816522 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:18:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: slow request 271.810665 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:18:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: slow request 271.805055 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:18:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: slow request 271.799345 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:18:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:33 smithi040 ceph-mon[97954]: slow request 271.793603 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:18:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: pgmap v195: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: 1767 slow requests, 5 included below; oldest blocked for > 272.160839 secs 2023-07-14T11:18:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: slow request 271.816522 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:18:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: slow request 271.810665 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:18:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: slow request 271.805055 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:18:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: slow request 271.799345 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:18:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:33 smithi045 ceph-mon[128791]: slow request 271.793603 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:18:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: 1765 slow requests, 5 included below; oldest blocked for > 276.229394 secs 2023-07-14T11:18:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: slow request 274.842787 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: slow request 274.817139 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: slow request 274.788796 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: slow request 274.761745 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: slow request 274.734985 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:35 smithi130 ceph-mon[129405]: pgmap v196: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: 1765 slow requests, 5 included below; oldest blocked for > 276.229394 secs 2023-07-14T11:18:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: slow request 274.842787 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: slow request 274.817139 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: slow request 274.788796 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: slow request 274.761745 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: slow request 274.734985 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:35 smithi045 ceph-mon[128791]: pgmap v196: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:35.834 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: 1765 slow requests, 5 included below; oldest blocked for > 276.229394 secs 2023-07-14T11:18:35.834 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: slow request 274.842787 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.834 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: slow request 274.817139 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.834 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: slow request 274.788796 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.835 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: slow request 274.761745 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.835 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: slow request 274.734985 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:35.835 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:35 smithi040 ceph-mon[97954]: pgmap v196: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:37 smithi130 ceph-mon[129405]: pgmap v197: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:37 smithi045 ceph-mon[128791]: pgmap v197: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: 1756 slow requests, 5 included below; oldest blocked for > 277.160955 secs 2023-07-14T11:18:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: slow request 276.788061 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:18:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: slow request 276.782319 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:18:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: slow request 276.776648 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:18:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: slow request 276.770883 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:18:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:38 smithi130 ceph-mon[129405]: slow request 276.764863 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:18:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: 1756 slow requests, 5 included below; oldest blocked for > 277.160955 secs 2023-07-14T11:18:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: slow request 276.788061 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:18:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: slow request 276.782319 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:18:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: slow request 276.776648 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:18:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: slow request 276.770883 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:18:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:38 smithi045 ceph-mon[128791]: slow request 276.764863 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:18:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:39 smithi130 ceph-mon[129405]: pgmap v198: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:39 smithi045 ceph-mon[128791]: pgmap v198: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:40.494 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:40.494 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:40.494 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:37 smithi040 ceph-mon[97954]: pgmap v197: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:40.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: 1756 slow requests, 5 included below; oldest blocked for > 277.160955 secs 2023-07-14T11:18:40.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: slow request 276.788061 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:18:40.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: slow request 276.782319 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:18:40.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: slow request 276.776648 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:18:40.495 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: slow request 276.770883 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:18:40.496 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:38 smithi040 ceph-mon[97954]: slow request 276.764863 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:18:40.496 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:40.496 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:40.496 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:39 smithi040 ceph-mon[97954]: pgmap v198: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: 1754 slow requests, 5 included below; oldest blocked for > 281.240873 secs 2023-07-14T11:18:40.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: slow request 279.714380 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: slow request 279.688652 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: slow request 279.663392 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: slow request 279.638339 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:40 smithi130 ceph-mon[129405]: slow request 279.609907 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: 1754 slow requests, 5 included below; oldest blocked for > 281.240873 secs 2023-07-14T11:18:40.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: slow request 279.714380 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: slow request 279.688652 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: slow request 279.663392 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: slow request 279.638339 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:40 smithi040 ceph-mon[97954]: slow request 279.609907 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: 1754 slow requests, 5 included below; oldest blocked for > 281.240873 secs 2023-07-14T11:18:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: slow request 279.714380 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: slow request 279.688652 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: slow request 279.663392 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: slow request 279.638339 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:40.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:40 smithi045 ceph-mon[128791]: slow request 279.609907 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:41 smithi130 ceph-mon[129405]: pgmap v199: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:41 smithi040 ceph-mon[97954]: pgmap v199: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:41 smithi045 ceph-mon[128791]: pgmap v199: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:43.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: pgmap v200: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:43.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: 1745 slow requests, 5 included below; oldest blocked for > 282.161015 secs 2023-07-14T11:18:43.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: slow request 281.758896 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:18:43.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: slow request 281.753007 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:18:43.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: slow request 281.747053 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:18:43.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: slow request 281.741099 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:18:43.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:43 smithi130 ceph-mon[129405]: slow request 281.735222 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:18:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: pgmap v200: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:43.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: 1745 slow requests, 5 included below; oldest blocked for > 282.161015 secs 2023-07-14T11:18:43.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: slow request 281.758896 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:18:43.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: slow request 281.753007 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:18:43.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: slow request 281.747053 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:18:43.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: slow request 281.741099 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:18:43.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:43 smithi040 ceph-mon[97954]: slow request 281.735222 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:18:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: pgmap v200: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: 1745 slow requests, 5 included below; oldest blocked for > 282.161015 secs 2023-07-14T11:18:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: slow request 281.758896 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:18:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: slow request 281.753007 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:18:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: slow request 281.747053 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:18:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: slow request 281.741099 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:18:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:43 smithi045 ceph-mon[128791]: slow request 281.735222 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:18:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: 1743 slow requests, 5 included below; oldest blocked for > 286.245965 secs 2023-07-14T11:18:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: slow request 284.575952 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: slow request 284.545447 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: slow request 284.519258 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: slow request 284.492817 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: slow request 284.466824 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:45.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:45 smithi045 ceph-mon[128791]: pgmap v201: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: 1743 slow requests, 5 included below; oldest blocked for > 286.245965 secs 2023-07-14T11:18:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: slow request 284.575952 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: slow request 284.545447 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: slow request 284.519258 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: slow request 284.492817 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: slow request 284.466824 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:45 smithi130 ceph-mon[129405]: pgmap v201: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: 1743 slow requests, 5 included below; oldest blocked for > 286.245965 secs 2023-07-14T11:18:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: slow request 284.575952 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: slow request 284.545447 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: slow request 284.519258 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: slow request 284.492817 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: slow request 284.466824 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:45 smithi040 ceph-mon[97954]: pgmap v201: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:47.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:47 smithi045 ceph-mon[128791]: pgmap v202: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:47 smithi130 ceph-mon[129405]: pgmap v202: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:48.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:47 smithi040 ceph-mon[97954]: pgmap v202: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:48.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: 1734 slow requests, 5 included below; oldest blocked for > 287.161153 secs 2023-07-14T11:18:48.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: slow request 286.729466 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:18:48.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: slow request 286.723623 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:18:48.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: slow request 286.717753 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:18:48.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: slow request 286.711820 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:18:48.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:48 smithi045 ceph-mon[128791]: slow request 286.705936 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:18:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: 1734 slow requests, 5 included below; oldest blocked for > 287.161153 secs 2023-07-14T11:18:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: slow request 286.729466 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:18:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: slow request 286.723623 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:18:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: slow request 286.717753 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:18:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: slow request 286.711820 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:18:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:48 smithi130 ceph-mon[129405]: slow request 286.705936 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:18:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: 1734 slow requests, 5 included below; oldest blocked for > 287.161153 secs 2023-07-14T11:18:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: slow request 286.729466 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:18:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: slow request 286.723623 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:18:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: slow request 286.717753 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:18:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: slow request 286.711820 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:18:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:48 smithi040 ceph-mon[97954]: slow request 286.705936 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:18:49.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:49 smithi045 ceph-mon[128791]: pgmap v203: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:49 smithi130 ceph-mon[129405]: pgmap v203: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:49 smithi040 ceph-mon[97954]: pgmap v203: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: 1731 slow requests, 5 included below; oldest blocked for > 291.262919 secs 2023-07-14T11:18:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: slow request 289.457634 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: slow request 289.430917 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: slow request 289.402482 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: slow request 289.374900 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:50 smithi130 ceph-mon[129405]: slow request 289.347373 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: 1731 slow requests, 5 included below; oldest blocked for > 291.262919 secs 2023-07-14T11:18:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: slow request 289.457634 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: slow request 289.430917 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: slow request 289.402482 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: slow request 289.374900 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:50 smithi040 ceph-mon[97954]: slow request 289.347373 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: 1731 slow requests, 5 included below; oldest blocked for > 291.262919 secs 2023-07-14T11:18:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: slow request 289.457634 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: slow request 289.430917 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: slow request 289.402482 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: slow request 289.374900 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:50 smithi045 ceph-mon[128791]: slow request 289.347373 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:51 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:51 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:51 smithi130 ceph-mon[129405]: pgmap v204: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:51 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:51 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:51 smithi040 ceph-mon[97954]: pgmap v204: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:51 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:51 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:51 smithi045 ceph-mon[128791]: pgmap v204: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: pgmap v205: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: 1722 slow requests, 5 included below; oldest blocked for > 292.161290 secs 2023-07-14T11:18:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: slow request 291.700048 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:18:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: slow request 291.694142 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:18:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: slow request 291.688382 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:18:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: slow request 291.682434 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:18:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:53 smithi130 ceph-mon[129405]: slow request 291.676548 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:18:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: pgmap v205: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: 1722 slow requests, 5 included below; oldest blocked for > 292.161290 secs 2023-07-14T11:18:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: slow request 291.700048 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:18:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: slow request 291.694142 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:18:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: slow request 291.688382 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:18:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: slow request 291.682434 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:18:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:53 smithi040 ceph-mon[97954]: slow request 291.676548 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:18:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: pgmap v205: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: 1722 slow requests, 5 included below; oldest blocked for > 292.161290 secs 2023-07-14T11:18:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: slow request 291.700048 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:18:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: slow request 291.694142 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:18:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: slow request 291.688382 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:18:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: slow request 291.682434 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:18:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:53 smithi045 ceph-mon[128791]: slow request 291.676548 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:18:54.886 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:18:55.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 -- 172.21.15.40:0/2521342851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28105af0 msgr2=0x7f5b28105ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/2521342851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28105af0 0x7f5b28105ec0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f5b18009990 tx=0x7f5b18009ca0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 -- 172.21.15.40:0/2521342851 shutdown_connections 2023-07-14T11:18:55.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/2521342851 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b28100490 0x7f5b2810c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/2521342851 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28105af0 0x7f5b28105ec0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/2521342851 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b280ffae0 0x7f5b280fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 -- 172.21.15.40:0/2521342851 >> 172.21.15.40:0/2521342851 conn(0x7f5b280fb5e0 msgr2=0x7f5b280fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 -- 172.21.15.40:0/2521342851 shutdown_connections 2023-07-14T11:18:55.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.407+0000 7f5b2ddff700 1 -- 172.21.15.40:0/2521342851 wait complete. 2023-07-14T11:18:55.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.408+0000 7f5b2ddff700 1 Processor -- start 2023-07-14T11:18:55.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 -- start start 2023-07-14T11:18:55.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 0x7f5b28197e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b28105af0 0x7f5b2819c7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b2819ce00 con 0x7f5b28105af0 2023-07-14T11:18:55.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b2819cf70 con 0x7f5b28100490 2023-07-14T11:18:55.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b2ddff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5b2819d130 con 0x7f5b280ffae0 2023-07-14T11:18:55.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 0x7f5b28197e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58016/0 (socket says 172.21.15.40:58016) 2023-07-14T11:18:55.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 0x7f5b28197e20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52300/0 (socket says 172.21.15.40:52300) 2023-07-14T11:18:55.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.409+0000 7f5b26ffd700 1 -- 172.21.15.40:0/935772433 learned_addr learned my addr 172.21.15.40:0/935772433 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:55.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b27fff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b28105af0 0x7f5b2819c7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 -- 172.21.15.40:0/935772433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 msgr2=0x7f5b28197e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 0x7f5b28197e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 -- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b28105af0 msgr2=0x7f5b2819c7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b28105af0 0x7f5b2819c7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 -- 172.21.15.40:0/935772433 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5b180095f0 con 0x7f5b28100490 2023-07-14T11:18:55.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c00e9a0 tx=0x7f5b1c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b24ff9700 1 -- 172.21.15.40:0/935772433 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c009790 con 0x7f5b28100490 2023-07-14T11:18:55.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b24ff9700 1 -- 172.21.15.40:0/935772433 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b1c010430 con 0x7f5b28100490 2023-07-14T11:18:55.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5b2819d410 con 0x7f5b28100490 2023-07-14T11:18:55.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.410+0000 7f5b24ff9700 1 -- 172.21.15.40:0/935772433 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c018680 con 0x7f5b28100490 2023-07-14T11:18:55.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.411+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b281a4c50 con 0x7f5b28100490 2023-07-14T11:18:55.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.415+0000 7f5b24ff9700 1 -- 172.21.15.40:0/935772433 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f5b1c0105a0 con 0x7f5b28100490 2023-07-14T11:18:55.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.417+0000 7f5b24ff9700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 0x7f5b100a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.417+0000 7f5b24ff9700 1 -- 172.21.15.40:0/935772433 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5b1c014070 con 0x7f5b28100490 2023-07-14T11:18:55.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.417+0000 7f5b277fe700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 0x7f5b100a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.418+0000 7f5b277fe700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 0x7f5b100a37f0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f5b1800b2e0 tx=0x7f5b18005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 Processor -- start 2023-07-14T11:18:55.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 -- start start 2023-07-14T11:18:55.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6e2d0 0x559abbc6e6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc7beb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbb74cb0 con 0x559abbc79ad0 2023-07-14T11:18:55.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbbc7a40 con 0x559abbc6e2d0 2023-07-14T11:18:55.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.421+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbbc0ed0 con 0x559abbc6ebe0 2023-07-14T11:18:55.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52308/0 (socket says 172.21.15.40:52308) 2023-07-14T11:18:55.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 -- 172.21.15.40:0/567956023 learned_addr learned my addr 172.21.15.40:0/567956023 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:55.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b277fe700 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6e2d0 0x559abbc6e6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b27fff700 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc7beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 -- 172.21.15.40:0/567956023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6e2d0 msgr2=0x559abbc6e6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6e2d0 0x559abbc6e6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 -- 172.21.15.40:0/567956023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 msgr2=0x559abbc7beb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc7beb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 -- 172.21.15.40:0/567956023 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559abb9eede0 con 0x559abbc6ebe0 2023-07-14T11:18:55.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.422+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0a1e60 tx=0x7f5b1c099b70 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2c5ae106536c102d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b0dffb700 1 -- 172.21.15.40:0/567956023 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c09df70 con 0x559abbc6ebe0 2023-07-14T11:18:55.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b0dffb700 1 -- 172.21.15.40:0/567956023 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b1c0a2390 con 0x559abbc6ebe0 2023-07-14T11:18:55.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b0dffb700 1 -- 172.21.15.40:0/567956023 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c099690 con 0x559abbc6ebe0 2023-07-14T11:18:55.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 -- 172.21.15.40:0/567956023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 msgr2=0x559abbc79590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0a1e60 tx=0x7f5b1c099b70 comp rx=0 tx=0).stop 2023-07-14T11:18:55.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 -- 172.21.15.40:0/567956023 shutdown_connections 2023-07-14T11:18:55.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc7beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6e2d0 0x559abbc6e6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/567956023 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6ebe0 0x559abbc79590 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.423+0000 7f5b4162bb80 1 -- 172.21.15.40:0/567956023 >> 172.21.15.40:0/567956023 conn(0x559abbc64da0 msgr2=0x559abbc6a6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.424+0000 7f5b4162bb80 1 -- 172.21.15.40:0/567956023 shutdown_connections 2023-07-14T11:18:55.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.424+0000 7f5b4162bb80 1 -- 172.21.15.40:0/567956023 wait complete. 2023-07-14T11:18:55.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.424+0000 7f5b4162bb80 1 -- start start 2023-07-14T11:18:55.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.424+0000 7f5b4162bb80 1 Processor -- start 2023-07-14T11:18:55.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc76d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc77280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbb60ac0 con 0x559abbc79ad0 2023-07-14T11:18:55.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbb607c0 con 0x559abbc6ebe0 2023-07-14T11:18:55.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abba44920 con 0x559abbc6e2d0 2023-07-14T11:18:55.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b4162bb80 10 client.0 _ll_register_callbacks cb 0x559abbb5e760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:18:55.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b277fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52320/0 (socket says 172.21.15.40:52320) 2023-07-14T11:18:55.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b277fe700 1 -- 172.21.15.40:0/467921165 learned_addr learned my addr 172.21.15.40:0/467921165 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:55.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc76d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.425+0000 7f5b27fff700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc77280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 -- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 msgr2=0x559abbc76d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc76d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 -- 172.21.15.40:0/467921165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 msgr2=0x559abbc77280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc77280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 -- 172.21.15.40:0/467921165 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b281a4c50 con 0x559abbc6e2d0 2023-07-14T11:18:55.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc76d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:18:55.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b277fe700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f5b18015000 tx=0x7f5b18020000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=5413abd25e9dbf30 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b06ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:18:55.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1802d040 con 0x559abbc6e2d0 2023-07-14T11:18:55.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559abbb1a4f0 con 0x559abbc6e2d0 2023-07-14T11:18:55.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.426+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5b1800f460 con 0x559abbc6e2d0 2023-07-14T11:18:55.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=92) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5b1800f620 con 0x559abbc6e2d0 2023-07-14T11:18:55.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 10 client.26338 fetch_fsmap learned FSMap version 92 2023-07-14T11:18:55.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=92,monmap=4+}) v3 -- 0x559abb9d7c60 con 0x559abbc6e2d0 2023-07-14T11:18:55.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559abbb1a4f0 con 0x559abbc6e2d0 2023-07-14T11:18:55.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 92) v1 ==== 7494+0+0 (secure 0 0 0) 0x7f5b18024740 con 0x559abbc6e2d0 2023-07-14T11:18:55.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 10 client.26338 fetch_fsmap finished waiting for FSMap version 92 2023-07-14T11:18:55.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 10 client.26338 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:18:55.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 20 client.26338 populate_metadata read hostname 'smithi040' 2023-07-14T11:18:55.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbc89440 0x559abbc8b850 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 4 client.26338 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:18:55.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559abbbb9eb0 con 0x559abbc89440 2023-07-14T11:18:55.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.427+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbc89440 0x559abbc8b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.428+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbc89440 0x559abbc8b850 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ac8372232169a543 server_cookie=ecb2590aaf21e176 in_seq=0 out_seq=0 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.428+0000 7f5b06ffd700 10 client.26338 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b06ffd700 1 -- 172.21.15.40:0/467921165 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x559abbbb9eb0 con 0x559abbc89440 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b06ffd700 10 client.26338 handle_command_reply: tid=0 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 client.26338 shutdown 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 msgr2=0x559abbc76800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f5b18015000 tx=0x7f5b18020000 comp rx=0 tx=0).stop 2023-07-14T11:18:55.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 shutdown_connections 2023-07-14T11:18:55.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbc89440 0x559abbc8b850 crc :-1 s=READY pgs=97 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc77280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b06ffd700 0 client.26338 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:55.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc76800 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/467921165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc76d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 >> 172.21.15.40:0/467921165 conn(0x559abbc64da0 msgr2=0x559abbc6a230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 shutdown_connections 2023-07-14T11:18:55.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.429+0000 7f5b4162bb80 1 -- 172.21.15.40:0/467921165 wait complete. 2023-07-14T11:18:55.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.430+0000 7f5b4162bb80 20 client.26338 trim_cache size 0 max 16384 2023-07-14T11:18:55.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.465+0000 7f5b4162bb80 1 Processor -- start 2023-07-14T11:18:55.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.465+0000 7f5b4162bb80 1 -- start start 2023-07-14T11:18:55.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc804f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc73fd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbc6f580 con 0x559abbc79ad0 2023-07-14T11:18:55.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbc6ea20 con 0x559abbc6ebe0 2023-07-14T11:18:55.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbc6f1c0 con 0x559abbc6e2d0 2023-07-14T11:18:55.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58052/0 (socket says 172.21.15.40:58052) 2023-07-14T11:18:55.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b26ffd700 1 -- 172.21.15.40:0/756908641 learned_addr learned my addr 172.21.15.40:0/756908641 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:55.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b27fff700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc73fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.466+0000 7f5b277fe700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc804f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 -- 172.21.15.40:0/756908641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 msgr2=0x559abbc804f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc804f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 -- 172.21.15.40:0/756908641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 msgr2=0x559abbc73fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc73fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 -- 172.21.15.40:0/756908641 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559abba15250 con 0x559abbc6ebe0 2023-07-14T11:18:55.472 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b27fff700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc73fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:18:55.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0963f0 tx=0x7f5b1c09a0c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8c96627317dba9fe server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b0e7fc700 1 -- 172.21.15.40:0/756908641 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c098360 con 0x559abbc6ebe0 2023-07-14T11:18:55.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b0e7fc700 1 -- 172.21.15.40:0/756908641 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b1c010880 con 0x559abbc6ebe0 2023-07-14T11:18:55.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.467+0000 7f5b0e7fc700 1 -- 172.21.15.40:0/756908641 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c0a2140 con 0x559abbc6ebe0 2023-07-14T11:18:55.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 -- 172.21.15.40:0/756908641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 msgr2=0x559abbc73a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0963f0 tx=0x7f5b1c09a0c0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 -- 172.21.15.40:0/756908641 shutdown_connections 2023-07-14T11:18:55.477 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc79ad0 0x559abbc73fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc6e2d0 0x559abbc804f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/756908641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbc73a90 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 -- 172.21.15.40:0/756908641 >> 172.21.15.40:0/756908641 conn(0x559abbc64da0 msgr2=0x559abbc71b30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.478 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 -- 172.21.15.40:0/756908641 shutdown_connections 2023-07-14T11:18:55.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.468+0000 7f5b4162bb80 1 -- 172.21.15.40:0/756908641 wait complete. 2023-07-14T11:18:55.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.469+0000 7f5b4162bb80 1 -- start start 2023-07-14T11:18:55.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.469+0000 7f5b4162bb80 1 Processor -- start 2023-07-14T11:18:55.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc6e2d0 0x559abbcd5670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.479 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc79ad0 0x559abbcd1290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbcd19d0 con 0x559abbc6e2d0 2023-07-14T11:18:55.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbcd1b40 con 0x559abbc6ebe0 2023-07-14T11:18:55.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559abbcd1d00 con 0x559abbc79ad0 2023-07-14T11:18:55.480 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b4162bb80 10 client.0 _ll_register_callbacks cb 0x7f5b18005350 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:18:55.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b26ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:58056/0 (socket says 172.21.15.40:58056) 2023-07-14T11:18:55.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b26ffd700 1 -- 172.21.15.40:0/45908486 learned_addr learned my addr 172.21.15.40:0/45908486 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:18:55.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b27fff700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc79ad0 0x559abbcd1290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.481 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.470+0000 7f5b277fe700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc6e2d0 0x559abbcd5670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 -- 172.21.15.40:0/45908486 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc79ad0 msgr2=0x559abbcd1290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc79ad0 0x559abbcd1290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 -- 172.21.15.40:0/45908486 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc6e2d0 msgr2=0x559abbcd5670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:18:55.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc6e2d0 0x559abbcd5670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.482 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 -- 172.21.15.40:0/45908486 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5b1c0b1040 con 0x559abbc6ebe0 2023-07-14T11:18:55.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b26ffd700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0969b0 tx=0x7f5b1c010ae0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=91cf64074627b25 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:18:55.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b06ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:18:55.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5b1c0b81b0 con 0x559abbc6ebe0 2023-07-14T11:18:55.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f5b1c0a2390 con 0x559abbc6ebe0 2023-07-14T11:18:55.483 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(93..93 src has 1..93) v4 ==== 9344+0+0 (secure 0 0 0) 0x7f5b1c0b8310 con 0x559abbc6ebe0 2023-07-14T11:18:55.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=92) v2 ==== 24+0+0 (secure 0 0 0) 0x7f5b1c0a1f00 con 0x559abbc6ebe0 2023-07-14T11:18:55.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b4162bb80 10 client.26381 fetch_fsmap learned FSMap version 92 2023-07-14T11:18:55.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.471+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=92,monmap=4+}) v3 -- 0x559abb9ed2b0 con 0x559abbc6ebe0 2023-07-14T11:18:55.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f5b1c0a2390 con 0x559abbc6ebe0 2023-07-14T11:18:55.484 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 92) v1 ==== 7494+0+0 (secure 0 0 0) 0x7f5b1c098ea0 con 0x559abbc6ebe0 2023-07-14T11:18:55.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 10 client.26381 fetch_fsmap finished waiting for FSMap version 92 2023-07-14T11:18:55.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 10 client.26381 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:18:55.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 20 client.26381 populate_metadata read hostname 'smithi040' 2023-07-14T11:18:55.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbcde0e0 0x559abbce04c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:18:55.485 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 4 client.26381 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x559abbb7c050 con 0x559abbcde0e0 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.472+0000 7f5b277fe700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbcde0e0 0x559abbce04c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b277fe700 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbcde0e0 0x559abbce04c0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d4846fed33924da2 server_cookie=544da7f71b674f78 in_seq=0 out_seq=0 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b06ffd700 10 client.26381 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b06ffd700 1 -- 172.21.15.40:0/45908486 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x559abbb7c050 con 0x559abbcde0e0 2023-07-14T11:18:55.486 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b06ffd700 10 client.26381 handle_command_reply: tid=0 2023-07-14T11:18:55.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b4162bb80 1 client.26381 shutdown 2023-07-14T11:18:55.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 msgr2=0x559abbcd5bb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c0969b0 tx=0x7f5b1c010ae0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 shutdown_connections 2023-07-14T11:18:55.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.473+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559abbcde0e0 0x559abbce04c0 crc :-1 s=READY pgs=98 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559abbc79ad0 0x559abbcd1290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b06ffd700 0 client.26381 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:18:55.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559abbc6e2d0 0x559abbcd5670 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 --2- 172.21.15.40:0/45908486 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559abbc6ebe0 0x559abbcd5bb0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 >> 172.21.15.40:0/45908486 conn(0x559abbc64da0 msgr2=0x559abbc703e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 shutdown_connections 2023-07-14T11:18:55.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 1 -- 172.21.15.40:0/45908486 wait complete. 2023-07-14T11:18:55.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b4162bb80 20 client.26381 trim_cache size 0 max 16384 2023-07-14T11:18:55.489 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:18:55.490 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:18:55.490 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:18:55.490 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:18:55.490 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:18:55.491 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:18:55.491 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:18:55.491 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:18:55.491 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:18:55.491 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:18:55.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 msgr2=0x7f5b100a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 0x7f5b100a37f0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f5b1800b2e0 tx=0x7f5b18005fb0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 msgr2=0x7f5b28198360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:18:55.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.474+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7f5b1c00e9a0 tx=0x7f5b1c00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 shutdown_connections 2023-07-14T11:18:55.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f5b100a1340 0x7f5b100a37f0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5b28105af0 0x7f5b2819c7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f5b280ffae0 0x7f5b28197e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 --2- 172.21.15.40:0/935772433 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f5b28100490 0x7f5b28198360 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:18:55.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 >> 172.21.15.40:0/935772433 conn(0x7f5b280fb5e0 msgr2=0x7f5b2810adf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:18:55.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 shutdown_connections 2023-07-14T11:18:55.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:18:55.475+0000 7f5b2ddff700 1 -- 172.21.15.40:0/935772433 wait complete. 2023-07-14T11:18:55.495 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:18:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: 1719 slow requests, 5 included below; oldest blocked for > 296.334413 secs 2023-07-14T11:18:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: slow request 294.393125 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: slow request 294.366532 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: slow request 294.339776 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: slow request 294.313646 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: slow request 294.287097 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:55 smithi130 ceph-mon[129405]: pgmap v206: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: 1719 slow requests, 5 included below; oldest blocked for > 296.334413 secs 2023-07-14T11:18:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: slow request 294.393125 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: slow request 294.366532 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: slow request 294.339776 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: slow request 294.313646 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: slow request 294.287097 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:55 smithi040 ceph-mon[97954]: pgmap v206: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: 1719 slow requests, 5 included below; oldest blocked for > 296.334413 secs 2023-07-14T11:18:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: slow request 294.393125 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: slow request 294.366532 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: slow request 294.339776 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: slow request 294.313646 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: slow request 294.287097 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:18:55.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:55 smithi045 ceph-mon[128791]: pgmap v206: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:18:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:57 smithi130 ceph-mon[129405]: pgmap v207: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:57 smithi040 ceph-mon[97954]: pgmap v207: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:18:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:57 smithi045 ceph-mon[128791]: pgmap v207: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: 1710 slow requests, 5 included below; oldest blocked for > 297.161456 secs 2023-07-14T11:18:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: slow request 296.670843 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:18:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: slow request 296.664926 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:18:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: slow request 296.659113 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:18:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: slow request 297.037392 seconds old, received at 2023-07-14T11:14:01.057620+0000: peer_request:mds.1:1191 currently dispatched 2023-07-14T11:18:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:58 smithi130 ceph-mon[129405]: slow request 297.030615 seconds old, received at 2023-07-14T11:14:01.064397+0000: peer_request:mds.1:1223 currently dispatched 2023-07-14T11:18:58.795 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: 1710 slow requests, 5 included below; oldest blocked for > 297.161456 secs 2023-07-14T11:18:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: slow request 296.670843 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:18:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: slow request 296.664926 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:18:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: slow request 296.659113 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:18:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: slow request 297.037392 seconds old, received at 2023-07-14T11:14:01.057620+0000: peer_request:mds.1:1191 currently dispatched 2023-07-14T11:18:58.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:58 smithi040 ceph-mon[97954]: slow request 297.030615 seconds old, received at 2023-07-14T11:14:01.064397+0000: peer_request:mds.1:1223 currently dispatched 2023-07-14T11:18:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: 1710 slow requests, 5 included below; oldest blocked for > 297.161456 secs 2023-07-14T11:18:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: slow request 296.670843 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:18:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: slow request 296.664926 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:18:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: slow request 296.659113 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:18:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: slow request 297.037392 seconds old, received at 2023-07-14T11:14:01.057620+0000: peer_request:mds.1:1191 currently dispatched 2023-07-14T11:18:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:58 smithi045 ceph-mon[128791]: slow request 297.030615 seconds old, received at 2023-07-14T11:14:01.064397+0000: peer_request:mds.1:1223 currently dispatched 2023-07-14T11:18:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:18:59 smithi130 ceph-mon[129405]: pgmap v208: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:18:59 smithi040 ceph-mon[97954]: pgmap v208: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:18:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:18:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:18:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:18:59 smithi045 ceph-mon[128791]: pgmap v208: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: 1707 slow requests, 5 included below; oldest blocked for > 301.514505 secs 2023-07-14T11:19:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: slow request 299.441151 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: slow request 299.414922 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: slow request 299.387100 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: slow request 300.948211 seconds old, received at 2023-07-14T11:13:58.668873+0000: client_request(mds.1:1206 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:00 smithi130 ceph-mon[129405]: slow request 300.919215 seconds old, received at 2023-07-14T11:13:58.697869+0000: client_request(mds.1:1238 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: 1707 slow requests, 5 included below; oldest blocked for > 301.514505 secs 2023-07-14T11:19:00.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: slow request 299.441151 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: slow request 299.414922 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: slow request 299.387100 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: slow request 300.948211 seconds old, received at 2023-07-14T11:13:58.668873+0000: client_request(mds.1:1206 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:00 smithi040 ceph-mon[97954]: slow request 300.919215 seconds old, received at 2023-07-14T11:13:58.697869+0000: client_request(mds.1:1238 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: 1707 slow requests, 5 included below; oldest blocked for > 301.514505 secs 2023-07-14T11:19:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: slow request 299.441151 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: slow request 299.414922 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: slow request 299.387100 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: slow request 300.948211 seconds old, received at 2023-07-14T11:13:58.668873+0000: client_request(mds.1:1206 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:00.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:00 smithi045 ceph-mon[128791]: slow request 300.919215 seconds old, received at 2023-07-14T11:13:58.697869+0000: client_request(mds.1:1238 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:01.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:01.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:01.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:01 smithi040 ceph-mon[97954]: pgmap v209: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:01 smithi045 ceph-mon[128791]: pgmap v209: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:02.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:01 smithi130 ceph-mon[129405]: pgmap v209: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: pgmap v210: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: 1700 slow requests, 5 included below; oldest blocked for > 302.161597 secs 2023-07-14T11:19:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: slow request 302.022164 seconds old, received at 2023-07-14T11:14:01.072989+0000: peer_request:mds.1:1255 currently dispatched 2023-07-14T11:19:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: slow request 302.016457 seconds old, received at 2023-07-14T11:14:01.078696+0000: peer_request:mds.1:1287 currently dispatched 2023-07-14T11:19:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: slow request 302.008204 seconds old, received at 2023-07-14T11:14:01.086949+0000: peer_request:mds.1:1319 currently dispatched 2023-07-14T11:19:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: slow request 301.998187 seconds old, received at 2023-07-14T11:14:01.096966+0000: peer_request:mds.1:1351 currently dispatched 2023-07-14T11:19:03.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:03 smithi045 ceph-mon[128791]: slow request 301.988164 seconds old, received at 2023-07-14T11:14:01.106989+0000: peer_request:mds.1:1383 currently dispatched 2023-07-14T11:19:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: pgmap v210: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: 1700 slow requests, 5 included below; oldest blocked for > 302.161597 secs 2023-07-14T11:19:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: slow request 302.022164 seconds old, received at 2023-07-14T11:14:01.072989+0000: peer_request:mds.1:1255 currently dispatched 2023-07-14T11:19:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: slow request 302.016457 seconds old, received at 2023-07-14T11:14:01.078696+0000: peer_request:mds.1:1287 currently dispatched 2023-07-14T11:19:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: slow request 302.008204 seconds old, received at 2023-07-14T11:14:01.086949+0000: peer_request:mds.1:1319 currently dispatched 2023-07-14T11:19:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: slow request 301.998187 seconds old, received at 2023-07-14T11:14:01.096966+0000: peer_request:mds.1:1351 currently dispatched 2023-07-14T11:19:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:03 smithi130 ceph-mon[129405]: slow request 301.988164 seconds old, received at 2023-07-14T11:14:01.106989+0000: peer_request:mds.1:1383 currently dispatched 2023-07-14T11:19:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: pgmap v210: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: 1700 slow requests, 5 included below; oldest blocked for > 302.161597 secs 2023-07-14T11:19:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: slow request 302.022164 seconds old, received at 2023-07-14T11:14:01.072989+0000: peer_request:mds.1:1255 currently dispatched 2023-07-14T11:19:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: slow request 302.016457 seconds old, received at 2023-07-14T11:14:01.078696+0000: peer_request:mds.1:1287 currently dispatched 2023-07-14T11:19:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: slow request 302.008204 seconds old, received at 2023-07-14T11:14:01.086949+0000: peer_request:mds.1:1319 currently dispatched 2023-07-14T11:19:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: slow request 301.998187 seconds old, received at 2023-07-14T11:14:01.096966+0000: peer_request:mds.1:1351 currently dispatched 2023-07-14T11:19:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:03 smithi040 ceph-mon[97954]: slow request 301.988164 seconds old, received at 2023-07-14T11:14:01.106989+0000: peer_request:mds.1:1383 currently dispatched 2023-07-14T11:19:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:05.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: 1697 slow requests, 5 included below; oldest blocked for > 306.525440 secs 2023-07-14T11:19:05.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: slow request 305.890451 seconds old, received at 2023-07-14T11:13:58.737569+0000: client_request(mds.1:1270 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:05.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: slow request 305.858673 seconds old, received at 2023-07-14T11:13:58.769346+0000: client_request(mds.1:1302 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:05.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: slow request 305.822777 seconds old, received at 2023-07-14T11:13:58.805242+0000: client_request(mds.1:1334 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:05.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: slow request 305.788981 seconds old, received at 2023-07-14T11:13:58.839038+0000: client_request(mds.1:1366 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:05.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: slow request 305.758174 seconds old, received at 2023-07-14T11:13:58.869846+0000: client_request(mds.1:1398 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:05.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:05 smithi045 ceph-mon[128791]: pgmap v211: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:19:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: 1697 slow requests, 5 included below; oldest blocked for > 306.525440 secs 2023-07-14T11:19:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: slow request 305.890451 seconds old, received at 2023-07-14T11:13:58.737569+0000: client_request(mds.1:1270 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: slow request 305.858673 seconds old, received at 2023-07-14T11:13:58.769346+0000: client_request(mds.1:1302 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: slow request 305.822777 seconds old, received at 2023-07-14T11:13:58.805242+0000: client_request(mds.1:1334 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: slow request 305.788981 seconds old, received at 2023-07-14T11:13:58.839038+0000: client_request(mds.1:1366 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: slow request 305.758174 seconds old, received at 2023-07-14T11:13:58.869846+0000: client_request(mds.1:1398 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:05 smithi130 ceph-mon[129405]: pgmap v211: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:19:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: 1697 slow requests, 5 included below; oldest blocked for > 306.525440 secs 2023-07-14T11:19:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: slow request 305.890451 seconds old, received at 2023-07-14T11:13:58.737569+0000: client_request(mds.1:1270 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: slow request 305.858673 seconds old, received at 2023-07-14T11:13:58.769346+0000: client_request(mds.1:1302 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: slow request 305.822777 seconds old, received at 2023-07-14T11:13:58.805242+0000: client_request(mds.1:1334 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: slow request 305.788981 seconds old, received at 2023-07-14T11:13:58.839038+0000: client_request(mds.1:1366 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: slow request 305.758174 seconds old, received at 2023-07-14T11:13:58.869846+0000: client_request(mds.1:1398 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:05 smithi040 ceph-mon[97954]: pgmap v211: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:19:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:07 smithi130 ceph-mon[129405]: pgmap v212: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:19:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:07 smithi040 ceph-mon[97954]: pgmap v212: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:19:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:07 smithi045 ceph-mon[128791]: pgmap v212: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:19:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: 1689 slow requests, 5 included below; oldest blocked for > 307.161724 secs 2023-07-14T11:19:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: slow request 306.978200 seconds old, received at 2023-07-14T11:14:01.117080+0000: peer_request:mds.1:1415 currently dispatched 2023-07-14T11:19:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: slow request 306.968198 seconds old, received at 2023-07-14T11:14:01.127082+0000: peer_request:mds.1:1447 currently dispatched 2023-07-14T11:19:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: slow request 306.958203 seconds old, received at 2023-07-14T11:14:01.137077+0000: peer_request:mds.1:1479 currently dispatched 2023-07-14T11:19:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: slow request 306.948032 seconds old, received at 2023-07-14T11:14:01.147248+0000: peer_request:mds.1:1511 currently dispatched 2023-07-14T11:19:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:08 smithi130 ceph-mon[129405]: slow request 306.938053 seconds old, received at 2023-07-14T11:14:01.157227+0000: peer_request:mds.1:1543 currently dispatched 2023-07-14T11:19:09.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: 1689 slow requests, 5 included below; oldest blocked for > 307.161724 secs 2023-07-14T11:19:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: slow request 306.978200 seconds old, received at 2023-07-14T11:14:01.117080+0000: peer_request:mds.1:1415 currently dispatched 2023-07-14T11:19:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: slow request 306.968198 seconds old, received at 2023-07-14T11:14:01.127082+0000: peer_request:mds.1:1447 currently dispatched 2023-07-14T11:19:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: slow request 306.958203 seconds old, received at 2023-07-14T11:14:01.137077+0000: peer_request:mds.1:1479 currently dispatched 2023-07-14T11:19:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: slow request 306.948032 seconds old, received at 2023-07-14T11:14:01.147248+0000: peer_request:mds.1:1511 currently dispatched 2023-07-14T11:19:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:08 smithi040 ceph-mon[97954]: slow request 306.938053 seconds old, received at 2023-07-14T11:14:01.157227+0000: peer_request:mds.1:1543 currently dispatched 2023-07-14T11:19:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: 1689 slow requests, 5 included below; oldest blocked for > 307.161724 secs 2023-07-14T11:19:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: slow request 306.978200 seconds old, received at 2023-07-14T11:14:01.117080+0000: peer_request:mds.1:1415 currently dispatched 2023-07-14T11:19:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: slow request 306.968198 seconds old, received at 2023-07-14T11:14:01.127082+0000: peer_request:mds.1:1447 currently dispatched 2023-07-14T11:19:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: slow request 306.958203 seconds old, received at 2023-07-14T11:14:01.137077+0000: peer_request:mds.1:1479 currently dispatched 2023-07-14T11:19:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: slow request 306.948032 seconds old, received at 2023-07-14T11:14:01.147248+0000: peer_request:mds.1:1511 currently dispatched 2023-07-14T11:19:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:08 smithi045 ceph-mon[128791]: slow request 306.938053 seconds old, received at 2023-07-14T11:14:01.157227+0000: peer_request:mds.1:1543 currently dispatched 2023-07-14T11:19:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:09 smithi130 ceph-mon[129405]: osdmap e94: 12 total, 12 up, 12 in 2023-07-14T11:19:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:09 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:09 smithi130 ceph-mon[129405]: pgmap v214: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:19:10.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:09 smithi040 ceph-mon[97954]: osdmap e94: 12 total, 12 up, 12 in 2023-07-14T11:19:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:09 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:09 smithi040 ceph-mon[97954]: pgmap v214: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:19:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:09 smithi045 ceph-mon[128791]: osdmap e94: 12 total, 12 up, 12 in 2023-07-14T11:19:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:09 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:09 smithi045 ceph-mon[128791]: pgmap v214: 129 pgs: 129 active+clean; 6.9 GiB data, 15 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:19:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: 1687 slow requests, 5 included below; oldest blocked for > 311.568525 secs 2023-07-14T11:19:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: slow request 310.771619 seconds old, received at 2023-07-14T11:13:58.899485+0000: client_request(mds.1:1430 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: slow request 310.737508 seconds old, received at 2023-07-14T11:13:58.933596+0000: client_request(mds.1:1462 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: slow request 310.694916 seconds old, received at 2023-07-14T11:13:58.976189+0000: client_request(mds.1:1494 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: slow request 310.659358 seconds old, received at 2023-07-14T11:13:59.011747+0000: client_request(mds.1:1526 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:10 smithi130 ceph-mon[129405]: slow request 310.625839 seconds old, received at 2023-07-14T11:13:59.045265+0000: client_request(mds.1:1558 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: 1687 slow requests, 5 included below; oldest blocked for > 311.568525 secs 2023-07-14T11:19:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: slow request 310.771619 seconds old, received at 2023-07-14T11:13:58.899485+0000: client_request(mds.1:1430 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: slow request 310.737508 seconds old, received at 2023-07-14T11:13:58.933596+0000: client_request(mds.1:1462 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: slow request 310.694916 seconds old, received at 2023-07-14T11:13:58.976189+0000: client_request(mds.1:1494 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: slow request 310.659358 seconds old, received at 2023-07-14T11:13:59.011747+0000: client_request(mds.1:1526 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:10 smithi040 ceph-mon[97954]: slow request 310.625839 seconds old, received at 2023-07-14T11:13:59.045265+0000: client_request(mds.1:1558 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: 1687 slow requests, 5 included below; oldest blocked for > 311.568525 secs 2023-07-14T11:19:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: slow request 310.771619 seconds old, received at 2023-07-14T11:13:58.899485+0000: client_request(mds.1:1430 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: slow request 310.737508 seconds old, received at 2023-07-14T11:13:58.933596+0000: client_request(mds.1:1462 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: slow request 310.694916 seconds old, received at 2023-07-14T11:13:58.976189+0000: client_request(mds.1:1494 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: slow request 310.659358 seconds old, received at 2023-07-14T11:13:59.011747+0000: client_request(mds.1:1526 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:10 smithi045 ceph-mon[128791]: slow request 310.625839 seconds old, received at 2023-07-14T11:13:59.045265+0000: client_request(mds.1:1558 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:11 smithi130 ceph-mon[129405]: pgmap v215: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:11 smithi040 ceph-mon[97954]: pgmap v215: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:11 smithi045 ceph-mon[128791]: pgmap v215: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: pgmap v216: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: 1679 slow requests, 5 included below; oldest blocked for > 312.161793 secs 2023-07-14T11:19:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: slow request 311.928130 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:19:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: slow request 311.920050 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:19:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: slow request 311.910189 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:19:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: slow request 311.900311 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:19:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:13 smithi130 ceph-mon[129405]: slow request 311.890297 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:19:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: pgmap v216: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: 1679 slow requests, 5 included below; oldest blocked for > 312.161793 secs 2023-07-14T11:19:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: slow request 311.928130 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:19:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: slow request 311.920050 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:19:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: slow request 311.910189 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:19:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: slow request 311.900311 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:19:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:13 smithi040 ceph-mon[97954]: slow request 311.890297 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:19:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: pgmap v216: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:19:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: 1679 slow requests, 5 included below; oldest blocked for > 312.161793 secs 2023-07-14T11:19:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: slow request 311.928130 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:19:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: slow request 311.920050 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:19:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: slow request 311.910189 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:19:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: slow request 311.900311 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:19:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:13 smithi045 ceph-mon[128791]: slow request 311.890297 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:19:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:14 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:14 smithi130 ceph-mon[129405]: osdmap e95: 12 total, 12 up, 12 in 2023-07-14T11:19:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:14 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:14 smithi040 ceph-mon[97954]: osdmap e95: 12 total, 12 up, 12 in 2023-07-14T11:19:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:14 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:14 smithi045 ceph-mon[128791]: osdmap e95: 12 total, 12 up, 12 in 2023-07-14T11:19:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: pgmap v218: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:19:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: 1675 slow requests, 5 included below; oldest blocked for > 316.586290 secs 2023-07-14T11:19:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: slow request 315.616567 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: slow request 315.583179 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: slow request 315.554052 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: slow request 315.520469 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:15 smithi130 ceph-mon[129405]: slow request 315.466374 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: pgmap v218: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:19:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: 1675 slow requests, 5 included below; oldest blocked for > 316.586290 secs 2023-07-14T11:19:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: slow request 315.616567 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: slow request 315.583179 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: slow request 315.554052 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: slow request 315.520469 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:15 smithi040 ceph-mon[97954]: slow request 315.466374 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: pgmap v218: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:19:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: 1675 slow requests, 5 included below; oldest blocked for > 316.586290 secs 2023-07-14T11:19:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: slow request 315.616567 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: slow request 315.583179 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: slow request 315.554052 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: slow request 315.520469 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:15 smithi045 ceph-mon[128791]: slow request 315.466374 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:17 smithi130 ceph-mon[129405]: pgmap v219: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:19:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:17 smithi040 ceph-mon[97954]: pgmap v219: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:19:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:17 smithi045 ceph-mon[128791]: pgmap v219: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:19:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: 1667 slow requests, 5 included below; oldest blocked for > 317.161878 secs 2023-07-14T11:19:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: slow request 316.880352 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:19:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: slow request 316.870256 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:19:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: slow request 316.860182 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:19:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: slow request 316.850180 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:19:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:18 smithi130 ceph-mon[129405]: slow request 316.840410 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:19:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: 1667 slow requests, 5 included below; oldest blocked for > 317.161878 secs 2023-07-14T11:19:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: slow request 316.880352 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:19:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: slow request 316.870256 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:19:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: slow request 316.860182 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:19:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: slow request 316.850180 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:19:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:18 smithi040 ceph-mon[97954]: slow request 316.840410 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:19:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: 1667 slow requests, 5 included below; oldest blocked for > 317.161878 secs 2023-07-14T11:19:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: slow request 316.880352 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:19:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: slow request 316.870256 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:19:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: slow request 316.860182 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:19:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: slow request 316.850180 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:19:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:18 smithi045 ceph-mon[128791]: slow request 316.840410 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:19:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:19 smithi130 ceph-mon[129405]: pgmap v220: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:19:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:19 smithi040 ceph-mon[97954]: pgmap v220: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:19:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:19 smithi045 ceph-mon[128791]: pgmap v220: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:19:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: 1664 slow requests, 5 included below; oldest blocked for > 321.684064 secs 2023-07-14T11:19:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: slow request 320.510426 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: slow request 320.484157 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: slow request 320.455877 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: slow request 320.428297 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:20 smithi130 ceph-mon[129405]: slow request 320.401748 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: 1664 slow requests, 5 included below; oldest blocked for > 321.684064 secs 2023-07-14T11:19:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: slow request 320.510426 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: slow request 320.484157 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: slow request 320.455877 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: slow request 320.428297 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:20 smithi040 ceph-mon[97954]: slow request 320.401748 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: 1664 slow requests, 5 included below; oldest blocked for > 321.684064 secs 2023-07-14T11:19:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: slow request 320.510426 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: slow request 320.484157 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: slow request 320.455877 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: slow request 320.428297 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:20 smithi045 ceph-mon[128791]: slow request 320.401748 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:21 smithi130 ceph-mon[129405]: pgmap v221: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:21 smithi040 ceph-mon[97954]: pgmap v221: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:21 smithi045 ceph-mon[128791]: pgmap v221: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: pgmap v222: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: 1655 slow requests, 5 included below; oldest blocked for > 322.161946 secs 2023-07-14T11:19:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: slow request 321.834751 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:19:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: slow request 321.828968 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:19:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: slow request 321.823296 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:19:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: slow request 321.817456 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:19:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:23 smithi130 ceph-mon[129405]: slow request 321.811597 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:19:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: pgmap v222: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: 1655 slow requests, 5 included below; oldest blocked for > 322.161946 secs 2023-07-14T11:19:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: slow request 321.834751 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:19:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: slow request 321.828968 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:19:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: slow request 321.823296 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:19:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: slow request 321.817456 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:19:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:23 smithi040 ceph-mon[97954]: slow request 321.811597 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:19:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: pgmap v222: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:19:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: 1655 slow requests, 5 included below; oldest blocked for > 322.161946 secs 2023-07-14T11:19:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: slow request 321.834751 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:19:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: slow request 321.828968 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:19:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: slow request 321.823296 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:19:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: slow request 321.817456 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:19:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:23 smithi045 ceph-mon[128791]: slow request 321.811597 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:19:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:24 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:24 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:24 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:24 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:24 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:24 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:25.497 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:19:25.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 -- 172.21.15.40:0/575158307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b201015b0 msgr2=0x7f3b20109aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 --2- 172.21.15.40:0/575158307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b201015b0 0x7f3b20109aa0 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7f3b14009990 tx=0x7f3b14009ca0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 -- 172.21.15.40:0/575158307 shutdown_connections 2023-07-14T11:19:25.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 --2- 172.21.15.40:0/575158307 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b2010a070 0x7f3b2010c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 --2- 172.21.15.40:0/575158307 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b20101070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 --2- 172.21.15.40:0/575158307 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b201015b0 0x7f3b20109aa0 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 -- 172.21.15.40:0/575158307 >> 172.21.15.40:0/575158307 conn(0x7f3b200fa4b0 msgr2=0x7f3b200fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 -- 172.21.15.40:0/575158307 shutdown_connections 2023-07-14T11:19:25.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.776+0000 7f3b25fff700 1 -- 172.21.15.40:0/575158307 wait complete. 2023-07-14T11:19:25.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.778+0000 7f3b25fff700 1 Processor -- start 2023-07-14T11:19:25.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.778+0000 7f3b25fff700 1 -- start start 2023-07-14T11:19:25.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.778+0000 7f3b25fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b200fe8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.778+0000 7f3b25fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b201015b0 0x7f3b200fee30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.778+0000 7f3b25fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b25fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b200ff660 con 0x7f3b201015b0 2023-07-14T11:19:25.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b25fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b201a4310 con 0x7f3b2010a070 2023-07-14T11:19:25.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b25fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b201a4450 con 0x7f3b20100ca0 2023-07-14T11:19:25.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56284/0 (socket says 172.21.15.40:56284) 2023-07-14T11:19:25.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1ffff700 1 -- 172.21.15.40:0/1255521131 learned_addr learned my addr 172.21.15.40:0/1255521131 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:25.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b200fe8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1effd700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b201015b0 0x7f3b200fee30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1ffff700 1 -- 172.21.15.40:0/1255521131 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 msgr2=0x7f3b200fe8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.779+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b200fe8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1ffff700 1 -- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b201015b0 msgr2=0x7f3b200fee30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b201015b0 0x7f3b200fee30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1ffff700 1 -- 172.21.15.40:0/1255521131 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b140095f0 con 0x7f3b2010a070 2023-07-14T11:19:25.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b200fe8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:19:25.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f3b1800ef30 tx=0x7f3b1800c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1cff9700 1 -- 172.21.15.40:0/1255521131 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b18018070 con 0x7f3b2010a070 2023-07-14T11:19:25.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1cff9700 1 -- 172.21.15.40:0/1255521131 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b1800f040 con 0x7f3b2010a070 2023-07-14T11:19:25.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b201a4650 con 0x7f3b2010a070 2023-07-14T11:19:25.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b1cff9700 1 -- 172.21.15.40:0/1255521131 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b1800cd00 con 0x7f3b2010a070 2023-07-14T11:19:25.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.780+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b201a4b30 con 0x7f3b2010a070 2023-07-14T11:19:25.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.786+0000 7f3b399b9b80 1 Processor -- start 2023-07-14T11:19:25.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.786+0000 7f3b1cff9700 1 -- 172.21.15.40:0/1255521131 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3b18003a00 con 0x7f3b2010a070 2023-07-14T11:19:25.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.786+0000 7f3b399b9b80 1 -- start start 2023-07-14T11:19:25.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac436a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 0x56258ac4e590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac50eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ab49cb0 con 0x56258ac4ead0 2023-07-14T11:19:25.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ab9ca40 con 0x56258ac432d0 2023-07-14T11:19:25.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ab95ed0 con 0x56258ac43be0 2023-07-14T11:19:25.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b1effd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac436a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b1effd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac436a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56298/0 (socket says 172.21.15.40:56298) 2023-07-14T11:19:25.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b1effd700 1 -- 172.21.15.40:0/674939805 learned_addr learned my addr 172.21.15.40:0/674939805 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:25.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.787+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 0x56258ac4e590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1effd700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac50eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/674939805 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 msgr2=0x56258ac436a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac436a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/674939805 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 msgr2=0x56258ac50eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac50eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/674939805 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56258a9c3de0 con 0x56258ac43be0 2023-07-14T11:19:25.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1effd700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac50eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:19:25.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 0x56258ac4e590 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f3b100114f0 tx=0x7f3b10011800 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c0cf85626d410a97 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b1cff9700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 0x7f3b080a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.788+0000 7f3b05ffb700 1 -- 172.21.15.40:0/674939805 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b10004230 con 0x56258ac43be0 2023-07-14T11:19:25.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.789+0000 7f3b1cff9700 1 -- 172.21.15.40:0/1255521131 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3b180ccb20 con 0x7f3b2010a070 2023-07-14T11:19:25.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.789+0000 7f3b05ffb700 1 -- 172.21.15.40:0/674939805 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b10004cf0 con 0x56258ac43be0 2023-07-14T11:19:25.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.789+0000 7f3b05ffb700 1 -- 172.21.15.40:0/674939805 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b10005630 con 0x56258ac43be0 2023-07-14T11:19:25.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.789+0000 7f3b1effd700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 0x7f3b080a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 -- 172.21.15.40:0/674939805 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 msgr2=0x56258ac4e590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 0x56258ac4e590 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 crypto rx=0x7f3b100114f0 tx=0x7f3b10011800 comp rx=0 tx=0).stop 2023-07-14T11:19:25.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b1effd700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 0x7f3b080a3770 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f3b14009990 tx=0x7f3b1400b280 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 -- 172.21.15.40:0/674939805 shutdown_connections 2023-07-14T11:19:25.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac50eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac436a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/674939805 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac43be0 0x56258ac4e590 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 -- 172.21.15.40:0/674939805 >> 172.21.15.40:0/674939805 conn(0x56258ac39da0 msgr2=0x56258ac3f6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 -- 172.21.15.40:0/674939805 shutdown_connections 2023-07-14T11:19:25.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.790+0000 7f3b399b9b80 1 -- 172.21.15.40:0/674939805 wait complete. 2023-07-14T11:19:25.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.791+0000 7f3b399b9b80 1 -- start start 2023-07-14T11:19:25.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.791+0000 7f3b399b9b80 1 Processor -- start 2023-07-14T11:19:25.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac4bcf0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac4ead0 0x56258ac4c230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ab35ac0 con 0x56258ac43be0 2023-07-14T11:19:25.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ab357c0 con 0x56258ac4ead0 2023-07-14T11:19:25.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258aa19920 con 0x56258ac432d0 2023-07-14T11:19:25.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b399b9b80 10 client.0 _ll_register_callbacks cb 0x56258ab33760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:19:25.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:36348/0 (socket says 172.21.15.40:36348) 2023-07-14T11:19:25.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.792+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/2236070127 learned_addr learned my addr 172.21.15.40:0/2236070127 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:25.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1effd700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac4bcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac4ead0 0x56258ac4c230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac4ead0 msgr2=0x56258ac4c230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac4ead0 0x56258ac4c230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/2236070127 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 msgr2=0x56258ac4bcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac4bcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/2236070127 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b10009c90 con 0x56258ac432d0 2023-07-14T11:19:25.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f3b10000c00 tx=0x7f3b10005090 comp rx=0 tx=0).ready entity=mon.1 client_cookie=685ba6ec90eaf927 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.793+0000 7f3afeffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:19:25.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b1000e470 con 0x56258ac432d0 2023-07-14T11:19:25.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3b1000e5d0 con 0x56258ac432d0 2023-07-14T11:19:25.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56258aaef4f0 con 0x56258ac432d0 2023-07-14T11:19:25.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=108) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3b1000e790 con 0x56258ac432d0 2023-07-14T11:19:25.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3b399b9b80 10 client.26356 fetch_fsmap learned FSMap version 108 2023-07-14T11:19:25.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.794+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=108,monmap=4+}) v3 -- 0x56258a9acc60 con 0x56258ac432d0 2023-07-14T11:19:25.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56258aaef4f0 con 0x56258ac432d0 2023-07-14T11:19:25.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 108) v1 ==== 7474+0+0 (secure 0 0 0) 0x7f3b100199c0 con 0x56258ac432d0 2023-07-14T11:19:25.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 10 client.26356 fetch_fsmap finished waiting for FSMap version 108 2023-07-14T11:19:25.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 10 client.26356 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:19:25.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 20 client.26356 populate_metadata read hostname 'smithi040' 2023-07-14T11:19:25.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258ac5e320 0x56258ac60730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 4 client.26356 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:19:25.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56258ab8eeb0 con 0x56258ac5e320 2023-07-14T11:19:25.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.795+0000 7f3b1effd700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258ac5e320 0x56258ac60730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.796+0000 7f3b1effd700 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258ac5e320 0x56258ac60730 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=4e2a9ace6589806 server_cookie=a154b454fac5cf0 in_seq=0 out_seq=0 2023-07-14T11:19:25.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.796+0000 7f3afeffd700 10 client.26356 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:25.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3afeffd700 1 -- 172.21.15.40:0/2236070127 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x56258ab8eeb0 con 0x56258ac5e320 2023-07-14T11:19:25.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3afeffd700 10 client.26356 handle_command_reply: tid=0 2023-07-14T11:19:25.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 client.26356 shutdown 2023-07-14T11:19:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 msgr2=0x56258ac4b7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 crypto rx=0x7f3b10000c00 tx=0x7f3b10005090 comp rx=0 tx=0).stop 2023-07-14T11:19:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 shutdown_connections 2023-07-14T11:19:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258ac5e320 0x56258ac60730 crc :-1 s=READY pgs=99 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac4ead0 0x56258ac4c230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4b7b0 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/2236070127 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac4bcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 >> 172.21.15.40:0/2236070127 conn(0x56258ac39da0 msgr2=0x56258ac3f290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.797+0000 7f3afeffd700 0 client.26356 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:25.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.798+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 shutdown_connections 2023-07-14T11:19:25.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.798+0000 7f3b399b9b80 1 -- 172.21.15.40:0/2236070127 wait complete. 2023-07-14T11:19:25.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.798+0000 7f3b399b9b80 20 client.26356 trim_cache size 0 max 16384 2023-07-14T11:19:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 Processor -- start 2023-07-14T11:19:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 -- start start 2023-07-14T11:19:25.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac43be0 0x56258ac458b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac45df0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ac404c0 con 0x56258ac4ead0 2023-07-14T11:19:25.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ac5ae10 con 0x56258ac43be0 2023-07-14T11:19:25.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.816+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ac40c70 con 0x56258ac432d0 2023-07-14T11:19:25.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:36358/0 (socket says 172.21.15.40:36358) 2023-07-14T11:19:25.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/494264461 learned_addr learned my addr 172.21.15.40:0/494264461 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:25.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1effd700 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac43be0 0x56258ac458b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac45df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/494264461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac43be0 msgr2=0x56258ac458b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac43be0 0x56258ac458b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/494264461 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 msgr2=0x56258ac45df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac45df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.817+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/494264461 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56258a9ea250 con 0x56258ac432d0 2023-07-14T11:19:25.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.818+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f3b10019cf0 tx=0x7f3b10009260 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f83593dfaa2c2f7d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.818+0000 7f3b067fc700 1 -- 172.21.15.40:0/494264461 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b10005100 con 0x56258ac432d0 2023-07-14T11:19:25.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.818+0000 7f3b067fc700 1 -- 172.21.15.40:0/494264461 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b10011b50 con 0x56258ac432d0 2023-07-14T11:19:25.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.818+0000 7f3b067fc700 1 -- 172.21.15.40:0/494264461 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b10024b90 con 0x56258ac432d0 2023-07-14T11:19:25.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- 172.21.15.40:0/494264461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 msgr2=0x56258ac4c060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7f3b10019cf0 tx=0x7f3b10009260 comp rx=0 tx=0).stop 2023-07-14T11:19:25.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- 172.21.15.40:0/494264461 shutdown_connections 2023-07-14T11:19:25.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac4ead0 0x56258ac45df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac432d0 0x56258ac4c060 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/494264461 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac43be0 0x56258ac458b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- 172.21.15.40:0/494264461 >> 172.21.15.40:0/494264461 conn(0x56258ac39da0 msgr2=0x56258ac451a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- 172.21.15.40:0/494264461 shutdown_connections 2023-07-14T11:19:25.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- 172.21.15.40:0/494264461 wait complete. 2023-07-14T11:19:25.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.819+0000 7f3b399b9b80 1 -- start start 2023-07-14T11:19:25.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 Processor -- start 2023-07-14T11:19:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac538d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 0x56258ac53e10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ac544e0 con 0x56258ac43be0 2023-07-14T11:19:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258ac54650 con 0x56258ac432d0 2023-07-14T11:19:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56258acb2210 con 0x56258ac4ead0 2023-07-14T11:19:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.820+0000 7f3b399b9b80 10 client.0 _ll_register_callbacks cb 0x7f3b180097c0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:19:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56322/0 (socket says 172.21.15.40:56322) 2023-07-14T11:19:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/3179543357 learned_addr learned my addr 172.21.15.40:0/3179543357 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 0x56258ac53e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1effd700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac538d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/3179543357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 msgr2=0x56258ac53e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 0x56258ac53e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/3179543357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 msgr2=0x56258ac538d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac538d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 -- 172.21.15.40:0/3179543357 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56258a9acc60 con 0x56258ac432d0 2023-07-14T11:19:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1ffff700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 0x56258ac53e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:19:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3b1f7fe700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f3b10024440 tx=0x7f3b10005780 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9186fd6424b1cbb9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.821+0000 7f3afeffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:19:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b10024810 con 0x56258ac432d0 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3b10004cf0 con 0x56258ac432d0 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3b10000e40 con 0x56258ac432d0 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=108) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3b10024970 con 0x56258ac432d0 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3b399b9b80 10 client.26399 fetch_fsmap learned FSMap version 108 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=108,monmap=4+}) v3 -- 0x56258a9c22b0 con 0x56258ac432d0 2023-07-14T11:19:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.822+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3b10004cf0 con 0x56258ac432d0 2023-07-14T11:19:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 108) v1 ==== 7474+0+0 (secure 0 0 0) 0x7f3b10036430 con 0x56258ac432d0 2023-07-14T11:19:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 10 client.26399 fetch_fsmap finished waiting for FSMap version 108 2023-07-14T11:19:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 10 client.26399 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:19:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 20 client.26399 populate_metadata read hostname 'smithi040' 2023-07-14T11:19:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258acb3440 0x56258acb5820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 4 client.26399 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:19:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56258ab51050 con 0x56258acb3440 2023-07-14T11:19:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b1effd700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258acb3440 0x56258acb5820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3b1effd700 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258acb3440 0x56258acb5820 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=af11186eda749dda server_cookie=40ec706ac1e0477a in_seq=0 out_seq=0 2023-07-14T11:19:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.823+0000 7f3afeffd700 10 client.26399 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:25.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3afeffd700 1 -- 172.21.15.40:0/3179543357 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x56258ab51050 con 0x56258acb3440 2023-07-14T11:19:25.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3afeffd700 10 client.26399 handle_command_reply: tid=0 2023-07-14T11:19:25.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 client.26399 shutdown 2023-07-14T11:19:25.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 msgr2=0x56258ac555a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f3b10024440 tx=0x7f3b10005780 comp rx=0 tx=0).stop 2023-07-14T11:19:25.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 shutdown_connections 2023-07-14T11:19:25.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56258acb3440 0x56258acb5820 crc :-1 s=READY pgs=100 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56258ac4ead0 0x56258ac53e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56258ac432d0 0x56258ac555a0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 --2- 172.21.15.40:0/3179543357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56258ac43be0 0x56258ac538d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 >> 172.21.15.40:0/3179543357 conn(0x56258ac39da0 msgr2=0x56258ac4a5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3afeffd700 0 client.26399 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:25.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.824+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 shutdown_connections 2023-07-14T11:19:25.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b399b9b80 1 -- 172.21.15.40:0/3179543357 wait complete. 2023-07-14T11:19:25.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b399b9b80 20 client.26399 trim_cache size 0 max 16384 2023-07-14T11:19:25.839 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:19:25.839 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:19:25.839 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:19:25.840 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:19:25.840 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:19:25.840 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:19:25.840 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:19:25.840 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:19:25.841 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:19:25.841 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:19:25.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 msgr2=0x7f3b080a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 0x7f3b080a3770 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f3b14009990 tx=0x7f3b1400b280 comp rx=0 tx=0).stop 2023-07-14T11:19:25.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 msgr2=0x7f3b201a3dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:25.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f3b1800ef30 tx=0x7f3b1800c3a0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 shutdown_connections 2023-07-14T11:19:25.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3b080a12c0 0x7f3b080a3770 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3b2010a070 0x7f3b201a3dd0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3b20100ca0 0x7f3b200fe8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 --2- 172.21.15.40:0/1255521131 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3b201015b0 0x7f3b200fee30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:25.843 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.825+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 >> 172.21.15.40:0/1255521131 conn(0x7f3b200fa4b0 msgr2=0x7f3b2010c170 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:25.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.826+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 shutdown_connections 2023-07-14T11:19:25.844 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:25.826+0000 7f3b25fff700 1 -- 172.21.15.40:0/1255521131 wait complete. 2023-07-14T11:19:25.844 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: pgmap v223: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:19:25.844 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: 1651 slow requests, 5 included below; oldest blocked for > 326.882464 secs 2023-07-14T11:19:25.845 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: slow request 325.573908 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:25.845 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: slow request 325.547351 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:25.845 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: slow request 325.521233 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:25.845 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: slow request 325.495020 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:25.846 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: slow request 325.469342 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:25.846 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:25 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:19:25.847 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:19:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: pgmap v223: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:19:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: 1651 slow requests, 5 included below; oldest blocked for > 326.882464 secs 2023-07-14T11:19:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: slow request 325.573908 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: slow request 325.547351 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: slow request 325.521233 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: slow request 325.495020 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: slow request 325.469342 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:25 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:19:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: pgmap v223: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:19:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: 1651 slow requests, 5 included below; oldest blocked for > 326.882464 secs 2023-07-14T11:19:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: slow request 325.573908 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: slow request 325.547351 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: slow request 325.521233 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: slow request 325.495020 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: slow request 325.469342 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:25 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:19:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:27 smithi130 ceph-mon[129405]: pgmap v224: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:27 smithi040 ceph-mon[97954]: pgmap v224: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:27 smithi045 ceph-mon[128791]: pgmap v224: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: 1644 slow requests, 5 included below; oldest blocked for > 327.162083 secs 2023-07-14T11:19:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: slow request 326.806132 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:19:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: slow request 326.800422 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:19:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: slow request 326.794650 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:19:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: slow request 326.789020 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:19:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:28 smithi130 ceph-mon[129405]: slow request 326.783279 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:19:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: 1644 slow requests, 5 included below; oldest blocked for > 327.162083 secs 2023-07-14T11:19:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: slow request 326.806132 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:19:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: slow request 326.800422 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:19:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: slow request 326.794650 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:19:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: slow request 326.789020 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:19:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:28 smithi040 ceph-mon[97954]: slow request 326.783279 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:19:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: 1644 slow requests, 5 included below; oldest blocked for > 327.162083 secs 2023-07-14T11:19:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: slow request 326.806132 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:19:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: slow request 326.800422 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:19:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: slow request 326.794650 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:19:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: slow request 326.789020 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:19:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:28 smithi045 ceph-mon[128791]: slow request 326.783279 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:19:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:29 smithi130 ceph-mon[129405]: pgmap v225: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:29 smithi040 ceph-mon[97954]: pgmap v225: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:29 smithi045 ceph-mon[128791]: pgmap v225: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: 1640 slow requests, 5 included below; oldest blocked for > 331.912147 secs 2023-07-14T11:19:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: slow request 330.470697 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: slow request 330.443702 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: slow request 330.416582 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: slow request 330.384861 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:30 smithi130 ceph-mon[129405]: slow request 330.359139 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: 1640 slow requests, 5 included below; oldest blocked for > 331.912147 secs 2023-07-14T11:19:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: slow request 330.470697 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: slow request 330.443702 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: slow request 330.416582 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: slow request 330.384861 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:30 smithi040 ceph-mon[97954]: slow request 330.359139 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: 1640 slow requests, 5 included below; oldest blocked for > 331.912147 secs 2023-07-14T11:19:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: slow request 330.470697 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: slow request 330.443702 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: slow request 330.416582 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: slow request 330.384861 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:30 smithi045 ceph-mon[128791]: slow request 330.359139 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:31 smithi130 ceph-mon[129405]: pgmap v226: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:31 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:19:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:31 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:19:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:31 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:19:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:31 smithi040 ceph-mon[97954]: pgmap v226: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:31 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:19:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:31 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:19:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:31 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:19:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:31 smithi045 ceph-mon[128791]: pgmap v226: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:31 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:19:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:31 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:19:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:31 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:19:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:32 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:32 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:32 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:32 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:32 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:32 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: pgmap v227: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: 1632 slow requests, 5 included below; oldest blocked for > 332.162196 secs 2023-07-14T11:19:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: slow request 331.777719 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:19:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: slow request 331.771938 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:19:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: slow request 331.765882 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:19:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: slow request 331.759869 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:19:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:33 smithi130 ceph-mon[129405]: slow request 331.754020 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:19:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: pgmap v227: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: 1632 slow requests, 5 included below; oldest blocked for > 332.162196 secs 2023-07-14T11:19:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: slow request 331.777719 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:19:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: slow request 331.771938 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:19:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: slow request 331.765882 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:19:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: slow request 331.759869 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:19:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:33 smithi040 ceph-mon[97954]: slow request 331.754020 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:19:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: pgmap v227: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: 1632 slow requests, 5 included below; oldest blocked for > 332.162196 secs 2023-07-14T11:19:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: slow request 331.777719 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:19:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: slow request 331.771938 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:19:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: slow request 331.765882 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:19:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: slow request 331.759869 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:19:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:33 smithi045 ceph-mon[128791]: slow request 331.754020 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:19:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: pgmap v228: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: 1628 slow requests, 5 included below; oldest blocked for > 336.959823 secs 2023-07-14T11:19:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: slow request 335.381567 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: slow request 335.356508 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: slow request 335.327162 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: slow request 335.288288 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:35 smithi130 ceph-mon[129405]: slow request 335.258488 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: pgmap v228: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: 1628 slow requests, 5 included below; oldest blocked for > 336.959823 secs 2023-07-14T11:19:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: slow request 335.381567 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: slow request 335.356508 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: slow request 335.327162 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: slow request 335.288288 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:35 smithi040 ceph-mon[97954]: slow request 335.258488 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: pgmap v228: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: 1628 slow requests, 5 included below; oldest blocked for > 336.959823 secs 2023-07-14T11:19:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: slow request 335.381567 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: slow request 335.356508 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: slow request 335.327162 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: slow request 335.288288 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:36.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:35 smithi045 ceph-mon[128791]: slow request 335.258488 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:37.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:37 smithi130 ceph-mon[129405]: pgmap v229: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:37 smithi040 ceph-mon[97954]: pgmap v229: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:37 smithi045 ceph-mon[128791]: pgmap v229: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: 1620 slow requests, 5 included below; oldest blocked for > 337.162260 secs 2023-07-14T11:19:38.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: slow request 336.748120 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:19:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: slow request 336.742131 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:19:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: slow request 336.736284 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:19:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: slow request 336.730397 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:19:38.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:38 smithi130 ceph-mon[129405]: slow request 336.724562 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:19:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: 1620 slow requests, 5 included below; oldest blocked for > 337.162260 secs 2023-07-14T11:19:38.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: slow request 336.748120 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:19:38.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: slow request 336.742131 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:19:38.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: slow request 336.736284 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:19:38.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: slow request 336.730397 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:19:38.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:38 smithi040 ceph-mon[97954]: slow request 336.724562 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:19:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: 1620 slow requests, 5 included below; oldest blocked for > 337.162260 secs 2023-07-14T11:19:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: slow request 336.748120 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:19:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: slow request 336.742131 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:19:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: slow request 336.736284 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:19:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: slow request 336.730397 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:19:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:38 smithi045 ceph-mon[128791]: slow request 336.724562 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:19:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:39.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:39.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:39 smithi130 ceph-mon[129405]: pgmap v230: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:39.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:39.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:39.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:39 smithi040 ceph-mon[97954]: pgmap v230: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:39 smithi045 ceph-mon[128791]: pgmap v230: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: 1615 slow requests, 5 included below; oldest blocked for > 342.103849 secs 2023-07-14T11:19:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: slow request 340.376332 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: slow request 340.349810 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: slow request 340.323891 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: slow request 340.297751 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: slow request 340.270937 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:41.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:41.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:41 smithi130 ceph-mon[129405]: pgmap v231: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: 1615 slow requests, 5 included below; oldest blocked for > 342.103849 secs 2023-07-14T11:19:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: slow request 340.376332 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: slow request 340.349810 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: slow request 340.323891 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: slow request 340.297751 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: slow request 340.270937 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:41.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:41.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:41 smithi040 ceph-mon[97954]: pgmap v231: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: 1615 slow requests, 5 included below; oldest blocked for > 342.103849 secs 2023-07-14T11:19:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: slow request 340.376332 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: slow request 340.349810 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: slow request 340.323891 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: slow request 340.297751 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: slow request 340.270937 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:41.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:41 smithi045 ceph-mon[128791]: pgmap v231: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: pgmap v232: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: 1608 slow requests, 5 included below; oldest blocked for > 342.162397 secs 2023-07-14T11:19:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: slow request 341.718802 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:19:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: slow request 341.712875 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:19:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: slow request 341.706992 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:19:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: slow request 341.700984 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:19:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:43 smithi045 ceph-mon[128791]: slow request 341.695066 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:19:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: pgmap v232: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: 1608 slow requests, 5 included below; oldest blocked for > 342.162397 secs 2023-07-14T11:19:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: slow request 341.718802 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:19:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: slow request 341.712875 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:19:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: slow request 341.706992 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:19:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: slow request 341.700984 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:19:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:43 smithi130 ceph-mon[129405]: slow request 341.695066 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:19:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: pgmap v232: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: 1608 slow requests, 5 included below; oldest blocked for > 342.162397 secs 2023-07-14T11:19:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: slow request 341.718802 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:19:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: slow request 341.712875 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:19:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: slow request 341.706992 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:19:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: slow request 341.700984 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:19:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:43 smithi040 ceph-mon[97954]: slow request 341.695066 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:19:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:45.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:45.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:45 smithi045 ceph-mon[128791]: pgmap v233: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:45 smithi130 ceph-mon[129405]: pgmap v233: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:45 smithi040 ceph-mon[97954]: pgmap v233: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: 1602 slow requests, 5 included below; oldest blocked for > 347.283637 secs 2023-07-14T11:19:46.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: slow request 345.422373 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: slow request 345.394706 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: slow request 345.367291 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: slow request 345.341380 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:46.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:46 smithi045 ceph-mon[128791]: slow request 345.314932 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: 1602 slow requests, 5 included below; oldest blocked for > 347.283637 secs 2023-07-14T11:19:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: slow request 345.422373 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: slow request 345.394706 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: slow request 345.367291 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: slow request 345.341380 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:46 smithi130 ceph-mon[129405]: slow request 345.314932 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: 1602 slow requests, 5 included below; oldest blocked for > 347.283637 secs 2023-07-14T11:19:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: slow request 345.422373 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: slow request 345.394706 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: slow request 345.367291 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: slow request 345.341380 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:47.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:46 smithi040 ceph-mon[97954]: slow request 345.314932 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:47 smithi130 ceph-mon[129405]: pgmap v234: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:47 smithi040 ceph-mon[97954]: pgmap v234: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:47 smithi045 ceph-mon[128791]: pgmap v234: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: 1595 slow requests, 5 included below; oldest blocked for > 347.162507 secs 2023-07-14T11:19:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: slow request 346.689410 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:19:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: slow request 346.683459 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:19:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: slow request 346.677570 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:19:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: slow request 346.671709 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:19:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:48 smithi130 ceph-mon[129405]: slow request 346.665778 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:19:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: 1595 slow requests, 5 included below; oldest blocked for > 347.162507 secs 2023-07-14T11:19:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: slow request 346.689410 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:19:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: slow request 346.683459 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:19:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: slow request 346.677570 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:19:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: slow request 346.671709 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:19:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:48 smithi040 ceph-mon[97954]: slow request 346.665778 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:19:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: 1595 slow requests, 5 included below; oldest blocked for > 347.162507 secs 2023-07-14T11:19:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: slow request 346.689410 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:19:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: slow request 346.683459 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:19:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: slow request 346.677570 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:19:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: slow request 346.671709 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:19:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:48 smithi045 ceph-mon[128791]: slow request 346.665778 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:19:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:49 smithi130 ceph-mon[129405]: pgmap v235: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:50.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:49 smithi040 ceph-mon[97954]: pgmap v235: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:49 smithi045 ceph-mon[128791]: pgmap v235: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: 1589 slow requests, 5 included below; oldest blocked for > 352.399470 secs 2023-07-14T11:19:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: slow request 350.403983 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: slow request 350.377887 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: slow request 350.351103 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: slow request 350.325312 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:50 smithi130 ceph-mon[129405]: slow request 350.299108 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: 1589 slow requests, 5 included below; oldest blocked for > 352.399470 secs 2023-07-14T11:19:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: slow request 350.403983 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: slow request 350.377887 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: slow request 350.351103 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: slow request 350.325312 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:50 smithi040 ceph-mon[97954]: slow request 350.299108 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: 1589 slow requests, 5 included below; oldest blocked for > 352.399470 secs 2023-07-14T11:19:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: slow request 350.403983 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: slow request 350.377887 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: slow request 350.351103 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: slow request 350.325312 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:50 smithi045 ceph-mon[128791]: slow request 350.299108 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:51 smithi130 ceph-mon[129405]: pgmap v236: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:52.045 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:51 smithi040 ceph-mon[97954]: pgmap v236: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:51 smithi045 ceph-mon[128791]: pgmap v236: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:52 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:53.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:52 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:52 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:53.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:52 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:52 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:53.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:52 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: pgmap v237: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: 1582 slow requests, 5 included below; oldest blocked for > 352.162613 secs 2023-07-14T11:19:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: slow request 351.660084 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:19:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: slow request 352.008922 seconds old, received at 2023-07-14T11:14:01.087247+0000: peer_request:mds.1:1320 currently dispatched 2023-07-14T11:19:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: slow request 351.998900 seconds old, received at 2023-07-14T11:14:01.097269+0000: peer_request:mds.1:1352 currently dispatched 2023-07-14T11:19:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: slow request 351.988881 seconds old, received at 2023-07-14T11:14:01.107288+0000: peer_request:mds.1:1384 currently dispatched 2023-07-14T11:19:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:53 smithi130 ceph-mon[129405]: slow request 351.978719 seconds old, received at 2023-07-14T11:14:01.117450+0000: peer_request:mds.1:1416 currently dispatched 2023-07-14T11:19:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: pgmap v237: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: 1582 slow requests, 5 included below; oldest blocked for > 352.162613 secs 2023-07-14T11:19:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: slow request 351.660084 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:19:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: slow request 352.008922 seconds old, received at 2023-07-14T11:14:01.087247+0000: peer_request:mds.1:1320 currently dispatched 2023-07-14T11:19:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: slow request 351.998900 seconds old, received at 2023-07-14T11:14:01.097269+0000: peer_request:mds.1:1352 currently dispatched 2023-07-14T11:19:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: slow request 351.988881 seconds old, received at 2023-07-14T11:14:01.107288+0000: peer_request:mds.1:1384 currently dispatched 2023-07-14T11:19:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:53 smithi040 ceph-mon[97954]: slow request 351.978719 seconds old, received at 2023-07-14T11:14:01.117450+0000: peer_request:mds.1:1416 currently dispatched 2023-07-14T11:19:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: pgmap v237: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: 1582 slow requests, 5 included below; oldest blocked for > 352.162613 secs 2023-07-14T11:19:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: slow request 351.660084 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:19:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: slow request 352.008922 seconds old, received at 2023-07-14T11:14:01.087247+0000: peer_request:mds.1:1320 currently dispatched 2023-07-14T11:19:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: slow request 351.998900 seconds old, received at 2023-07-14T11:14:01.097269+0000: peer_request:mds.1:1352 currently dispatched 2023-07-14T11:19:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: slow request 351.988881 seconds old, received at 2023-07-14T11:14:01.107288+0000: peer_request:mds.1:1384 currently dispatched 2023-07-14T11:19:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:53 smithi045 ceph-mon[128791]: slow request 351.978719 seconds old, received at 2023-07-14T11:14:01.117450+0000: peer_request:mds.1:1416 currently dispatched 2023-07-14T11:19:55.848 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:19:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: pgmap v238: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: 1576 slow requests, 5 included below; oldest blocked for > 357.474090 secs 2023-07-14T11:19:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: slow request 355.345435 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: slow request 356.770269 seconds old, received at 2023-07-14T11:13:58.806401+0000: client_request(mds.1:1335 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: slow request 356.736835 seconds old, received at 2023-07-14T11:13:58.839834+0000: client_request(mds.1:1367 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: slow request 356.705954 seconds old, received at 2023-07-14T11:13:58.870715+0000: client_request(mds.1:1399 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:55 smithi130 ceph-mon[129405]: slow request 356.675965 seconds old, received at 2023-07-14T11:13:58.900704+0000: client_request(mds.1:1431 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: pgmap v238: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: 1576 slow requests, 5 included below; oldest blocked for > 357.474090 secs 2023-07-14T11:19:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: slow request 355.345435 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: slow request 356.770269 seconds old, received at 2023-07-14T11:13:58.806401+0000: client_request(mds.1:1335 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: slow request 356.736835 seconds old, received at 2023-07-14T11:13:58.839834+0000: client_request(mds.1:1367 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: slow request 356.705954 seconds old, received at 2023-07-14T11:13:58.870715+0000: client_request(mds.1:1399 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:55 smithi040 ceph-mon[97954]: slow request 356.675965 seconds old, received at 2023-07-14T11:13:58.900704+0000: client_request(mds.1:1431 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: pgmap v238: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:19:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: 1576 slow requests, 5 included below; oldest blocked for > 357.474090 secs 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: slow request 355.345435 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: slow request 356.770269 seconds old, received at 2023-07-14T11:13:58.806401+0000: client_request(mds.1:1335 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: slow request 356.736835 seconds old, received at 2023-07-14T11:13:58.839834+0000: client_request(mds.1:1367 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: slow request 356.705954 seconds old, received at 2023-07-14T11:13:58.870715+0000: client_request(mds.1:1399 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:55 smithi045 ceph-mon[128791]: slow request 356.675965 seconds old, received at 2023-07-14T11:13:58.900704+0000: client_request(mds.1:1431 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:19:56.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 -- 172.21.15.40:0/209085012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 msgr2=0x7f229010be90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.108 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 --2- 172.21.15.40:0/209085012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f229010be90 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7f2280009990 tx=0x7f2280009ca0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 -- 172.21.15.40:0/209085012 shutdown_connections 2023-07-14T11:19:56.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 --2- 172.21.15.40:0/209085012 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f229010c3d0 0x7f229010e7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 --2- 172.21.15.40:0/209085012 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f22900fe820 0x7f22900febf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 --2- 172.21.15.40:0/209085012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f229010be90 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.109 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 -- 172.21.15.40:0/209085012 >> 172.21.15.40:0/209085012 conn(0x7f22900fa4b0 msgr2=0x7f22900fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 -- 172.21.15.40:0/209085012 shutdown_connections 2023-07-14T11:19:56.110 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.105+0000 7f22977ff700 1 -- 172.21.15.40:0/209085012 wait complete. 2023-07-14T11:19:56.111 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.106+0000 7f22977ff700 1 Processor -- start 2023-07-14T11:19:56.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.106+0000 7f22977ff700 1 -- start start 2023-07-14T11:19:56.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 0x7f2290197e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f22901983a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.112 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f229019ce40 con 0x7f22900fe820 2023-07-14T11:19:56.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f229019cfb0 con 0x7f229010c3d0 2023-07-14T11:19:56.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f22977ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f229019d170 con 0x7f22900ff1c0 2023-07-14T11:19:56.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f229559b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 0x7f2290197e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.113 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48210/0 (socket says 172.21.15.40:48210) 2023-07-14T11:19:56.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f22901983a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 -- 172.21.15.40:0/2271961442 learned_addr learned my addr 172.21.15.40:0/2271961442 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:56.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 -- 172.21.15.40:0/2271961442 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 msgr2=0x7f22901983a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f22901983a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.114 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 -- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 msgr2=0x7f2290197e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 0x7f2290197e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f2295d9c700 1 -- 172.21.15.40:0/2271961442 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f22800095f0 con 0x7f229010c3d0 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.107+0000 7f229559b700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 0x7f2290197e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f2288009fd0 tx=0x7f228800ef90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f22867fc700 1 -- 172.21.15.40:0/2271961442 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2288018070 con 0x7f229010c3d0 2023-07-14T11:19:56.115 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f22867fc700 1 -- 172.21.15.40:0/2271961442 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f228800f040 con 0x7f229010c3d0 2023-07-14T11:19:56.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f229019d3f0 con 0x7f229010c3d0 2023-07-14T11:19:56.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f22867fc700 1 -- 172.21.15.40:0/2271961442 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f228800cd00 con 0x7f229010c3d0 2023-07-14T11:19:56.116 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.108+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f22901a4c90 con 0x7f229010c3d0 2023-07-14T11:19:56.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.112+0000 7f22867fc700 1 -- 172.21.15.40:0/2271961442 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f2288004ad0 con 0x7f229010c3d0 2023-07-14T11:19:56.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.113+0000 7f22867fc700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 0x7f227c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.114+0000 7f22867fc700 1 -- 172.21.15.40:0/2271961442 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f22880ccaf0 con 0x7f229010c3d0 2023-07-14T11:19:56.117 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.114+0000 7f229559b700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 0x7f227c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.114+0000 7f229559b700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 0x7f227c0a3770 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f228c011490 tx=0x7f228c009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.115+0000 7f22ab1ddb80 1 Processor -- start 2023-07-14T11:19:56.118 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.115+0000 7f22ab1ddb80 1 -- start start 2023-07-14T11:19:56.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.115+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c6356a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c53bcb0 con 0x56162c6352d0 2023-07-14T11:19:56.120 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c58ea40 con 0x56162c640ad0 2023-07-14T11:19:56.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c587ed0 con 0x56162c635be0 2023-07-14T11:19:56.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f229559b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c6356a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.121 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:48218/0 (socket says 172.21.15.40:48218) 2023-07-14T11:19:56.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:42418/0 (socket says 172.21.15.40:42418) 2023-07-14T11:19:56.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 -- 172.21.15.40:0/2696177964 learned_addr learned my addr 172.21.15.40:0/2696177964 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:56.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 -- 172.21.15.40:0/2696177964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 msgr2=0x56162c640590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.122 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 -- 172.21.15.40:0/2696177964 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 msgr2=0x56162c6356a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c6356a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2295d9c700 1 -- 172.21.15.40:0/2696177964 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56162c3b5de0 con 0x56162c640ad0 2023-07-14T11:19:56.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.116+0000 7f2294d9a700 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:19:56.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.117+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f22880962d0 tx=0x7f2288095c70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f31da0156f7f123a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.123 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.117+0000 7f227b7fe700 1 -- 172.21.15.40:0/2696177964 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f22880991c0 con 0x56162c640ad0 2023-07-14T11:19:56.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.117+0000 7f227b7fe700 1 -- 172.21.15.40:0/2696177964 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2288099860 con 0x56162c640ad0 2023-07-14T11:19:56.124 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.117+0000 7f227b7fe700 1 -- 172.21.15.40:0/2696177964 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2288099ac0 con 0x56162c640ad0 2023-07-14T11:19:56.126 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2696177964 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 msgr2=0x56162c642eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.126 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f22880962d0 tx=0x7f2288095c70 comp rx=0 tx=0).stop 2023-07-14T11:19:56.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2696177964 shutdown_connections 2023-07-14T11:19:56.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c642eb0 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c6356a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2696177964 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c640590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2696177964 >> 172.21.15.40:0/2696177964 conn(0x56162c62bda0 msgr2=0x56162c6316a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2696177964 shutdown_connections 2023-07-14T11:19:56.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.118+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2696177964 wait complete. 2023-07-14T11:19:56.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.119+0000 7f22ab1ddb80 1 -- start start 2023-07-14T11:19:56.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.119+0000 7f22ab1ddb80 1 Processor -- start 2023-07-14T11:19:56.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c63d840 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c63dd80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c63e2c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c527ac0 con 0x56162c6352d0 2023-07-14T11:19:56.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c5277c0 con 0x56162c640ad0 2023-07-14T11:19:56.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c40b920 con 0x56162c635be0 2023-07-14T11:19:56.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f22ab1ddb80 10 client.0 _ll_register_callbacks cb 0x56162c525760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:19:56.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f229559b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c63d840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c63dd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f229559b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c63d840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:53158/0 (socket says 172.21.15.40:53158) 2023-07-14T11:19:56.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f229559b700 1 -- 172.21.15.40:0/3136053000 learned_addr learned my addr 172.21.15.40:0/3136053000 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.120+0000 7f2295d9c700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c63e2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 -- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 msgr2=0x56162c63e2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c63e2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 -- 172.21.15.40:0/3136053000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 msgr2=0x56162c63d840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c63d840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 -- 172.21.15.40:0/3136053000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f229010f3f0 con 0x56162c635be0 2023-07-14T11:19:56.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2295d9c700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c63e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:19:56.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2294d9a700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c63dd80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f228000b2e0 tx=0x7f2280005ad0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9e3abde5cc2a0455 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2278ff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:19:56.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f228001d070 con 0x56162c635be0 2023-07-14T11:19:56.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f2280004500 con 0x56162c635be0 2023-07-14T11:19:56.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.121+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56162c4e14f0 con 0x56162c635be0 2023-07-14T11:19:56.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=120) v2 ==== 24+0+0 (secure 0 0 0) 0x7f228002e890 con 0x56162c635be0 2023-07-14T11:19:56.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 10 client.26374 fetch_fsmap learned FSMap version 120 2023-07-14T11:19:56.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=120,monmap=4+}) v3 -- 0x7f22901a4c90 con 0x56162c635be0 2023-07-14T11:19:56.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56162c4e14f0 con 0x56162c635be0 2023-07-14T11:19:56.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 120) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f228000f460 con 0x56162c635be0 2023-07-14T11:19:56.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 10 client.26374 fetch_fsmap finished waiting for FSMap version 120 2023-07-14T11:19:56.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 10 client.26374 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:19:56.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 20 client.26374 populate_metadata read hostname 'smithi040' 2023-07-14T11:19:56.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c650530 0x56162c652940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 4 client.26374 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:19:56.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56162c580eb0 con 0x56162c650530 2023-07-14T11:19:56.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.122+0000 7f229559b700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c650530 0x56162c652940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.123+0000 7f2278ff9700 10 client.26374 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:56.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.123+0000 7f229559b700 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c650530 0x56162c652940 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=29ff2fba3930a824 server_cookie=f5c9f9e26744f13a in_seq=0 out_seq=0 2023-07-14T11:19:56.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f2278ff9700 1 -- 172.21.15.40:0/3136053000 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x56162c580eb0 con 0x56162c650530 2023-07-14T11:19:56.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f2278ff9700 10 client.26374 handle_command_reply: tid=0 2023-07-14T11:19:56.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 client.26374 shutdown 2023-07-14T11:19:56.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 msgr2=0x56162c63dd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c63dd80 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7f228000b2e0 tx=0x7f2280005ad0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 shutdown_connections 2023-07-14T11:19:56.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c650530 0x56162c652940 crc :-1 s=READY pgs=101 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c63e2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c6352d0 0x56162c63d840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f2278ff9700 0 client.26374 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:56.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.124+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/3136053000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c635be0 0x56162c63dd80 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.125+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 >> 172.21.15.40:0/3136053000 conn(0x56162c62bda0 msgr2=0x56162c631290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.125+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 shutdown_connections 2023-07-14T11:19:56.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.125+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/3136053000 wait complete. 2023-07-14T11:19:56.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.125+0000 7f22ab1ddb80 20 client.26374 trim_cache size 0 max 16384 2023-07-14T11:19:56.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 Processor -- start 2023-07-14T11:19:56.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 -- start start 2023-07-14T11:19:56.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 0x56162c63aab0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c640ad0 0x56162c63aff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c62dec0 con 0x56162c640ad0 2023-07-14T11:19:56.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c640440 con 0x56162c635be0 2023-07-14T11:19:56.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.157+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c636dd0 con 0x56162c6352d0 2023-07-14T11:19:56.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f229559b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 0x56162c63aab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f229559b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:42438/0 (socket says 172.21.15.40:42438) 2023-07-14T11:19:56.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f229559b700 1 -- 172.21.15.40:0/2092701123 learned_addr learned my addr 172.21.15.40:0/2092701123 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:56.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2295d9c700 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c640ad0 0x56162c63aff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 -- 172.21.15.40:0/2092701123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 msgr2=0x56162c6474b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 -- 172.21.15.40:0/2092701123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c640ad0 msgr2=0x56162c63aff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c640ad0 0x56162c63aff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 -- 172.21.15.40:0/2092701123 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56162c62d620 con 0x56162c635be0 2023-07-14T11:19:56.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f229559b700 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:19:56.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.158+0000 7f2294d9a700 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 0x56162c63aab0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f228000c010 tx=0x7f2280009ca0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=3bb882b0ae3a2e9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f227bfff700 1 -- 172.21.15.40:0/2092701123 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f228001d070 con 0x56162c635be0 2023-07-14T11:19:56.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f227bfff700 1 -- 172.21.15.40:0/2092701123 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f228003c030 con 0x56162c635be0 2023-07-14T11:19:56.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f227bfff700 1 -- 172.21.15.40:0/2092701123 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f228002c600 con 0x56162c635be0 2023-07-14T11:19:56.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2092701123 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 msgr2=0x56162c63aab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 0x56162c63aab0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7f228000c010 tx=0x7f2280009ca0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2092701123 shutdown_connections 2023-07-14T11:19:56.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c640ad0 0x56162c63aff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c6474b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/2092701123 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c635be0 0x56162c63aab0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.159+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2092701123 >> 172.21.15.40:0/2092701123 conn(0x56162c62bda0 msgr2=0x56162c638b30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.160+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2092701123 shutdown_connections 2023-07-14T11:19:56.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.160+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/2092701123 wait complete. 2023-07-14T11:19:56.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.160+0000 7f22ab1ddb80 1 -- start start 2023-07-14T11:19:56.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.160+0000 7f22ab1ddb80 1 Processor -- start 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c635be0 0x56162c69cc70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c698340 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c698a50 con 0x56162c635be0 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c698bc0 con 0x56162c640ad0 2023-07-14T11:19:56.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56162c698d80 con 0x56162c6352d0 2023-07-14T11:19:56.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f22ab1ddb80 10 client.0 _ll_register_callbacks cb 0x7f228000bb60 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:19:56.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f2294d9a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c635be0 0x56162c69cc70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:42450/0 (socket says 172.21.15.40:42450) 2023-07-14T11:19:56.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 -- 172.21.15.40:0/1226884944 learned_addr learned my addr 172.21.15.40:0/1226884944 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:19:56.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f2295d9c700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c698340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 -- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 msgr2=0x56162c698340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c698340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 -- 172.21.15.40:0/1226884944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c635be0 msgr2=0x56162c69cc70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:19:56.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c635be0 0x56162c69cc70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.161+0000 7f229559b700 1 -- 172.21.15.40:0/1226884944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f228c017070 con 0x56162c6352d0 2023-07-14T11:19:56.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f229559b700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f228c004650 tx=0x7f228c0047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=29087939b27c2482 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:19:56.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f2278ff9700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:19:56.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f228c024040 con 0x56162c6352d0 2023-07-14T11:19:56.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f2288099860 con 0x56162c6352d0 2023-07-14T11:19:56.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(95..95 src has 1..95) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f228c029590 con 0x56162c6352d0 2023-07-14T11:19:56.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=120) v2 ==== 24+0+0 (secure 0 0 0) 0x7f228c029950 con 0x56162c6352d0 2023-07-14T11:19:56.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f22ab1ddb80 10 client.26380 fetch_fsmap learned FSMap version 120 2023-07-14T11:19:56.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.162+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=120,monmap=4+}) v3 -- 0x7f22901a4c90 con 0x56162c6352d0 2023-07-14T11:19:56.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f2288099860 con 0x56162c6352d0 2023-07-14T11:19:56.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 120) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f228c032b60 con 0x56162c6352d0 2023-07-14T11:19:56.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 10 client.26380 fetch_fsmap finished waiting for FSMap version 120 2023-07-14T11:19:56.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 10 client.26380 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:19:56.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 20 client.26380 populate_metadata read hostname 'smithi040' 2023-07-14T11:19:56.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c6a55e0 0x56162c6a79f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:19:56.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 4 client.26380 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:19:56.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56162c543050 con 0x56162c6a55e0 2023-07-14T11:19:56.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.163+0000 7f2294d9a700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c6a55e0 0x56162c6a79f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:19:56.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f2294d9a700 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c6a55e0 0x56162c6a79f0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=504da5ee725c5b69 server_cookie=eaace3bbce86898d in_seq=0 out_seq=0 2023-07-14T11:19:56.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f2278ff9700 10 client.26380 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:56.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f2278ff9700 1 -- 172.21.15.40:0/1226884944 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x56162c543050 con 0x56162c6a55e0 2023-07-14T11:19:56.177 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f2278ff9700 10 client.26380 handle_command_reply: tid=0 2023-07-14T11:19:56.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f22ab1ddb80 1 client.26380 shutdown 2023-07-14T11:19:56.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 msgr2=0x56162c69c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.164+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7f228c004650 tx=0x7f228c0047c0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 shutdown_connections 2023-07-14T11:19:56.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56162c6a55e0 0x56162c6a79f0 crc :-1 s=READY pgs=102 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56162c640ad0 0x56162c698340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56162c6352d0 0x56162c69c730 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f2278ff9700 0 client.26380 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:19:56.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 --2- 172.21.15.40:0/1226884944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56162c635be0 0x56162c69cc70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 >> 172.21.15.40:0/1226884944 conn(0x56162c62bda0 msgr2=0x56162c63f150 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 shutdown_connections 2023-07-14T11:19:56.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 1 -- 172.21.15.40:0/1226884944 wait complete. 2023-07-14T11:19:56.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22ab1ddb80 20 client.26380 trim_cache size 0 max 16384 2023-07-14T11:19:56.180 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:19:56.181 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:19:56.181 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:19:56.181 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:19:56.181 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:19:56.182 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:19:56.182 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:19:56.182 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:19:56.182 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:19:56.182 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:19:56.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 msgr2=0x7f227c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 0x7f227c0a3770 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f228c011490 tx=0x7f228c009260 comp rx=0 tx=0).stop 2023-07-14T11:19:56.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 msgr2=0x7f229019c800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:19:56.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.165+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f2288009fd0 tx=0x7f228800ef90 comp rx=0 tx=0).stop 2023-07-14T11:19:56.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 shutdown_connections 2023-07-14T11:19:56.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f227c0a12c0 0x7f227c0a3770 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f229010c3d0 0x7f229019c800 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f22900fe820 0x7f2290197e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 --2- 172.21.15.40:0/2271961442 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f22900ff1c0 0x7f22901983a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:19:56.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 >> 172.21.15.40:0/2271961442 conn(0x7f22900fa4b0 msgr2=0x7f229010d6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:19:56.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 shutdown_connections 2023-07-14T11:19:56.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:19:56.166+0000 7f22977ff700 1 -- 172.21.15.40:0/2271961442 wait complete. 2023-07-14T11:19:56.187 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:19:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:57.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:57 smithi130 ceph-mon[129405]: pgmap v239: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:57.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:57 smithi040 ceph-mon[97954]: pgmap v239: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:19:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:57.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:57 smithi045 ceph-mon[128791]: pgmap v239: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: 1569 slow requests, 5 included below; oldest blocked for > 357.162722 secs 2023-07-14T11:19:58.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: slow request 356.968877 seconds old, received at 2023-07-14T11:14:01.127402+0000: peer_request:mds.1:1448 currently dispatched 2023-07-14T11:19:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: slow request 356.958888 seconds old, received at 2023-07-14T11:14:01.137390+0000: peer_request:mds.1:1480 currently dispatched 2023-07-14T11:19:58.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: slow request 356.948682 seconds old, received at 2023-07-14T11:14:01.147596+0000: peer_request:mds.1:1512 currently dispatched 2023-07-14T11:19:58.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: slow request 356.938732 seconds old, received at 2023-07-14T11:14:01.157547+0000: peer_request:mds.1:1544 currently dispatched 2023-07-14T11:19:58.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:58 smithi130 ceph-mon[129405]: slow request 356.928740 seconds old, received at 2023-07-14T11:14:01.167538+0000: peer_request:mds.1:1576 currently dispatched 2023-07-14T11:19:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: 1569 slow requests, 5 included below; oldest blocked for > 357.162722 secs 2023-07-14T11:19:58.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: slow request 356.968877 seconds old, received at 2023-07-14T11:14:01.127402+0000: peer_request:mds.1:1448 currently dispatched 2023-07-14T11:19:58.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: slow request 356.958888 seconds old, received at 2023-07-14T11:14:01.137390+0000: peer_request:mds.1:1480 currently dispatched 2023-07-14T11:19:58.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: slow request 356.948682 seconds old, received at 2023-07-14T11:14:01.147596+0000: peer_request:mds.1:1512 currently dispatched 2023-07-14T11:19:58.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: slow request 356.938732 seconds old, received at 2023-07-14T11:14:01.157547+0000: peer_request:mds.1:1544 currently dispatched 2023-07-14T11:19:58.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:58 smithi040 ceph-mon[97954]: slow request 356.928740 seconds old, received at 2023-07-14T11:14:01.167538+0000: peer_request:mds.1:1576 currently dispatched 2023-07-14T11:19:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: 1569 slow requests, 5 included below; oldest blocked for > 357.162722 secs 2023-07-14T11:19:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: slow request 356.968877 seconds old, received at 2023-07-14T11:14:01.127402+0000: peer_request:mds.1:1448 currently dispatched 2023-07-14T11:19:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: slow request 356.958888 seconds old, received at 2023-07-14T11:14:01.137390+0000: peer_request:mds.1:1480 currently dispatched 2023-07-14T11:19:58.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: slow request 356.948682 seconds old, received at 2023-07-14T11:14:01.147596+0000: peer_request:mds.1:1512 currently dispatched 2023-07-14T11:19:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: slow request 356.938732 seconds old, received at 2023-07-14T11:14:01.157547+0000: peer_request:mds.1:1544 currently dispatched 2023-07-14T11:19:58.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:58 smithi045 ceph-mon[128791]: slow request 356.928740 seconds old, received at 2023-07-14T11:14:01.167538+0000: peer_request:mds.1:1576 currently dispatched 2023-07-14T11:19:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:59.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:19:59 smithi130 ceph-mon[129405]: pgmap v240: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:59.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:19:59 smithi040 ceph-mon[97954]: pgmap v240: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:19:59.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:19:59.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:19:59.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:19:59 smithi045 ceph-mon[128791]: pgmap v240: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:00 smithi045 ceph-mon[128791]: Health detail: HEALTH_WARN 2 MDSs report slow requests 2023-07-14T11:20:00.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:00 smithi045 ceph-mon[128791]: [WRN] MDS_SLOW_REQUEST: 2 MDSs report slow requests 2023-07-14T11:20:00.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:00 smithi045 ceph-mon[128791]: mds.a(mds.1): 1576 slow requests are blocked > 180 secs 2023-07-14T11:20:00.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:00 smithi045 ceph-mon[128791]: mds.b(mds.0): 1582 slow requests are blocked > 180 secs 2023-07-14T11:20:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:00 smithi130 ceph-mon[129405]: Health detail: HEALTH_WARN 2 MDSs report slow requests 2023-07-14T11:20:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:00 smithi130 ceph-mon[129405]: [WRN] MDS_SLOW_REQUEST: 2 MDSs report slow requests 2023-07-14T11:20:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:00 smithi130 ceph-mon[129405]: mds.a(mds.1): 1576 slow requests are blocked > 180 secs 2023-07-14T11:20:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:00 smithi130 ceph-mon[129405]: mds.b(mds.0): 1582 slow requests are blocked > 180 secs 2023-07-14T11:20:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:00 smithi040 ceph-mon[97954]: Health detail: HEALTH_WARN 2 MDSs report slow requests 2023-07-14T11:20:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:00 smithi040 ceph-mon[97954]: [WRN] MDS_SLOW_REQUEST: 2 MDSs report slow requests 2023-07-14T11:20:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:00 smithi040 ceph-mon[97954]: mds.a(mds.1): 1576 slow requests are blocked > 180 secs 2023-07-14T11:20:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:00 smithi040 ceph-mon[97954]: mds.b(mds.0): 1582 slow requests are blocked > 180 secs 2023-07-14T11:20:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:01.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:01.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:01 smithi045 ceph-mon[128791]: pgmap v241: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:01 smithi130 ceph-mon[129405]: pgmap v241: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:01 smithi040 ceph-mon[97954]: pgmap v241: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:02.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: 1563 slow requests, 5 included below; oldest blocked for > 362.579265 secs 2023-07-14T11:20:02.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: slow request 361.878834 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:02.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: slow request 361.844388 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:02.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: slow request 361.813842 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:02.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: slow request 361.784810 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:02.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:02 smithi045 ceph-mon[128791]: slow request 361.749910 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: 1563 slow requests, 5 included below; oldest blocked for > 362.579265 secs 2023-07-14T11:20:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: slow request 361.878834 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: slow request 361.844388 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: slow request 361.813842 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: slow request 361.784810 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:02 smithi130 ceph-mon[129405]: slow request 361.749910 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: 1563 slow requests, 5 included below; oldest blocked for > 362.579265 secs 2023-07-14T11:20:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: slow request 361.878834 seconds old, received at 2023-07-14T11:13:58.803011+0000: client_request(mds.1:1332 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: slow request 361.844388 seconds old, received at 2023-07-14T11:13:58.837457+0000: client_request(mds.1:1364 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: slow request 361.813842 seconds old, received at 2023-07-14T11:13:58.868003+0000: client_request(mds.1:1396 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: slow request 361.784810 seconds old, received at 2023-07-14T11:13:58.897035+0000: client_request(mds.1:1428 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:02 smithi040 ceph-mon[97954]: slow request 361.749910 seconds old, received at 2023-07-14T11:13:58.931935+0000: client_request(mds.1:1460 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:03.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: pgmap v242: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: 1558 slow requests, 5 included below; oldest blocked for > 362.162802 secs 2023-07-14T11:20:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: slow request 362.000012 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:20:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: slow request 361.989994 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:20:03.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: slow request 361.979891 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:20:03.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: slow request 361.969892 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:20:03.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:03 smithi045 ceph-mon[128791]: slow request 361.959924 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:20:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: pgmap v242: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: 1558 slow requests, 5 included below; oldest blocked for > 362.162802 secs 2023-07-14T11:20:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: slow request 362.000012 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:20:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: slow request 361.989994 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:20:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: slow request 361.979891 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:20:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: slow request 361.969892 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:20:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:03 smithi130 ceph-mon[129405]: slow request 361.959924 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:20:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: pgmap v242: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: 1558 slow requests, 5 included below; oldest blocked for > 362.162802 secs 2023-07-14T11:20:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: slow request 362.000012 seconds old, received at 2023-07-14T11:14:01.096347+0000: peer_request:mds.1:1349 currently dispatched 2023-07-14T11:20:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: slow request 361.989994 seconds old, received at 2023-07-14T11:14:01.106365+0000: peer_request:mds.1:1381 currently dispatched 2023-07-14T11:20:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: slow request 361.979891 seconds old, received at 2023-07-14T11:14:01.116468+0000: peer_request:mds.1:1413 currently dispatched 2023-07-14T11:20:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: slow request 361.969892 seconds old, received at 2023-07-14T11:14:01.126466+0000: peer_request:mds.1:1445 currently dispatched 2023-07-14T11:20:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:03 smithi040 ceph-mon[97954]: slow request 361.959924 seconds old, received at 2023-07-14T11:14:01.136434+0000: peer_request:mds.1:1477 currently dispatched 2023-07-14T11:20:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:05 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:05 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:05 smithi130 ceph-mon[129405]: pgmap v243: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:20:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:05 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:05 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:05 smithi040 ceph-mon[97954]: pgmap v243: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:20:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:05 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:05 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:05 smithi045 ceph-mon[128791]: pgmap v243: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:20:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: 1551 slow requests, 5 included below; oldest blocked for > 367.668293 secs 2023-07-14T11:20:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: slow request 366.798124 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: slow request 366.761251 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: slow request 366.727687 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: slow request 366.700721 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: slow request 366.667271 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:08.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:08.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:07 smithi130 ceph-mon[129405]: pgmap v244: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:20:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: 1551 slow requests, 5 included below; oldest blocked for > 367.668293 secs 2023-07-14T11:20:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: slow request 366.798124 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: slow request 366.761251 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: slow request 366.727687 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: slow request 366.700721 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: slow request 366.667271 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:08.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:07 smithi040 ceph-mon[97954]: pgmap v244: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:20:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: 1551 slow requests, 5 included below; oldest blocked for > 367.668293 secs 2023-07-14T11:20:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: slow request 366.798124 seconds old, received at 2023-07-14T11:13:58.972749+0000: client_request(mds.1:1492 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: slow request 366.761251 seconds old, received at 2023-07-14T11:13:59.009621+0000: client_request(mds.1:1524 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: slow request 366.727687 seconds old, received at 2023-07-14T11:13:59.043185+0000: client_request(mds.1:1556 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: slow request 366.700721 seconds old, received at 2023-07-14T11:13:59.070151+0000: client_request(mds.1:1588 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: slow request 366.667271 seconds old, received at 2023-07-14T11:13:59.103601+0000: client_request(mds.1:1620 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:08.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:08.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:08.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:07 smithi045 ceph-mon[128791]: pgmap v244: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 23 op/s 2023-07-14T11:20:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: 1546 slow requests, 5 included below; oldest blocked for > 367.162890 secs 2023-07-14T11:20:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: slow request 366.949931 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:20:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: slow request 366.939842 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:20:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: slow request 366.929834 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:20:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: slow request 366.921597 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:20:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:08 smithi130 ceph-mon[129405]: slow request 366.911917 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:20:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: 1546 slow requests, 5 included below; oldest blocked for > 367.162890 secs 2023-07-14T11:20:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: slow request 366.949931 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:20:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: slow request 366.939842 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:20:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: slow request 366.929834 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:20:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: slow request 366.921597 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:20:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:08 smithi040 ceph-mon[97954]: slow request 366.911917 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:20:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: 1546 slow requests, 5 included below; oldest blocked for > 367.162890 secs 2023-07-14T11:20:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: slow request 366.949931 seconds old, received at 2023-07-14T11:14:01.146516+0000: peer_request:mds.1:1509 currently dispatched 2023-07-14T11:20:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: slow request 366.939842 seconds old, received at 2023-07-14T11:14:01.156604+0000: peer_request:mds.1:1541 currently dispatched 2023-07-14T11:20:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: slow request 366.929834 seconds old, received at 2023-07-14T11:14:01.166612+0000: peer_request:mds.1:1573 currently dispatched 2023-07-14T11:20:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: slow request 366.921597 seconds old, received at 2023-07-14T11:14:01.174849+0000: peer_request:mds.1:1605 currently dispatched 2023-07-14T11:20:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:08 smithi045 ceph-mon[128791]: slow request 366.911917 seconds old, received at 2023-07-14T11:14:01.184530+0000: peer_request:mds.1:1637 currently dispatched 2023-07-14T11:20:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:09 smithi130 ceph-mon[129405]: osdmap e96: 12 total, 12 up, 12 in 2023-07-14T11:20:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:09 smithi130 ceph-mon[129405]: pgmap v246: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:20:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:09 smithi040 ceph-mon[97954]: osdmap e96: 12 total, 12 up, 12 in 2023-07-14T11:20:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:09 smithi040 ceph-mon[97954]: pgmap v246: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:20:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:09 smithi045 ceph-mon[128791]: osdmap e96: 12 total, 12 up, 12 in 2023-07-14T11:20:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:09 smithi045 ceph-mon[128791]: pgmap v246: 129 pgs: 129 active+clean; 5.9 GiB data, 13 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:20:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: pgmap v247: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: 1539 slow requests, 5 included below; oldest blocked for > 372.781674 secs 2023-07-14T11:20:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: slow request 371.751522 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: slow request 371.718956 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: slow request 371.667414 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: slow request 371.609888 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:11 smithi130 ceph-mon[129405]: slow request 371.583361 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: pgmap v247: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: 1539 slow requests, 5 included below; oldest blocked for > 372.781674 secs 2023-07-14T11:20:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: slow request 371.751522 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: slow request 371.718956 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: slow request 371.667414 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: slow request 371.609888 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:11 smithi040 ceph-mon[97954]: slow request 371.583361 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: pgmap v247: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: 1539 slow requests, 5 included below; oldest blocked for > 372.781674 secs 2023-07-14T11:20:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: slow request 371.751522 seconds old, received at 2023-07-14T11:13:59.132732+0000: client_request(mds.1:1652 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: slow request 371.718956 seconds old, received at 2023-07-14T11:13:59.165298+0000: client_request(mds.1:1684 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: slow request 371.667414 seconds old, received at 2023-07-14T11:13:59.216840+0000: client_request(mds.1:1716 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: slow request 371.609888 seconds old, received at 2023-07-14T11:13:59.274366+0000: client_request(mds.1:1748 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:11 smithi045 ceph-mon[128791]: slow request 371.583361 seconds old, received at 2023-07-14T11:13:59.300893+0000: client_request(mds.1:1780 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: pgmap v248: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: 1533 slow requests, 5 included below; oldest blocked for > 372.162955 secs 2023-07-14T11:20:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: slow request 371.902110 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:20:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: slow request 371.892087 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:20:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: slow request 371.882038 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:20:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: slow request 371.871936 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:20:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:13 smithi130 ceph-mon[129405]: slow request 371.861898 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:20:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: pgmap v248: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: 1533 slow requests, 5 included below; oldest blocked for > 372.162955 secs 2023-07-14T11:20:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: slow request 371.902110 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:20:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: slow request 371.892087 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:20:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: slow request 371.882038 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:20:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: slow request 371.871936 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:20:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:13 smithi040 ceph-mon[97954]: slow request 371.861898 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:20:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: pgmap v248: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:20:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: 1533 slow requests, 5 included below; oldest blocked for > 372.162955 secs 2023-07-14T11:20:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: slow request 371.902110 seconds old, received at 2023-07-14T11:14:01.194401+0000: peer_request:mds.1:1669 currently dispatched 2023-07-14T11:20:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: slow request 371.892087 seconds old, received at 2023-07-14T11:14:01.204424+0000: peer_request:mds.1:1701 currently dispatched 2023-07-14T11:20:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: slow request 371.882038 seconds old, received at 2023-07-14T11:14:01.214473+0000: peer_request:mds.1:1733 currently dispatched 2023-07-14T11:20:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: slow request 371.871936 seconds old, received at 2023-07-14T11:14:01.224576+0000: peer_request:mds.1:1765 currently dispatched 2023-07-14T11:20:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:13 smithi045 ceph-mon[128791]: slow request 371.861898 seconds old, received at 2023-07-14T11:14:01.234613+0000: peer_request:mds.1:1797 currently dispatched 2023-07-14T11:20:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:15 smithi130 ceph-mon[129405]: osdmap e97: 12 total, 12 up, 12 in 2023-07-14T11:20:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:15 smithi130 ceph-mon[129405]: pgmap v250: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:20:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:15 smithi040 ceph-mon[97954]: osdmap e97: 12 total, 12 up, 12 in 2023-07-14T11:20:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:15 smithi040 ceph-mon[97954]: pgmap v250: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:20:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:15.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:15 smithi045 ceph-mon[128791]: osdmap e97: 12 total, 12 up, 12 in 2023-07-14T11:20:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:15 smithi045 ceph-mon[128791]: pgmap v250: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:20:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: 1527 slow requests, 5 included below; oldest blocked for > 377.929603 secs 2023-07-14T11:20:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: slow request 376.703057 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: slow request 376.675459 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: slow request 376.648921 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: slow request 376.622652 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: slow request 376.596111 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:17.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:17.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:17 smithi130 ceph-mon[129405]: pgmap v251: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 253 B/s wr, 12 op/s 2023-07-14T11:20:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: 1527 slow requests, 5 included below; oldest blocked for > 377.929603 secs 2023-07-14T11:20:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: slow request 376.703057 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: slow request 376.675459 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: slow request 376.648921 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: slow request 376.622652 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: slow request 376.596111 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:17.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:17.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:17 smithi040 ceph-mon[97954]: pgmap v251: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 253 B/s wr, 12 op/s 2023-07-14T11:20:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: 1527 slow requests, 5 included below; oldest blocked for > 377.929603 secs 2023-07-14T11:20:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: slow request 376.703057 seconds old, received at 2023-07-14T11:13:59.329126+0000: client_request(mds.1:1812 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: slow request 376.675459 seconds old, received at 2023-07-14T11:13:59.356723+0000: client_request(mds.1:1844 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: slow request 376.648921 seconds old, received at 2023-07-14T11:13:59.383261+0000: client_request(mds.1:1876 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: slow request 376.622652 seconds old, received at 2023-07-14T11:13:59.409530+0000: client_request(mds.1:1908 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: slow request 376.596111 seconds old, received at 2023-07-14T11:13:59.436072+0000: client_request(mds.1:1940 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:17.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:17.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:17.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:17 smithi045 ceph-mon[128791]: pgmap v251: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.4 KiB/s rd, 253 B/s wr, 12 op/s 2023-07-14T11:20:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: 1522 slow requests, 5 included below; oldest blocked for > 377.163020 secs 2023-07-14T11:20:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: slow request 376.851954 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:20:18.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: slow request 376.841906 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:20:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: slow request 376.836164 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:20:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: slow request 376.830389 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:20:18.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:18 smithi130 ceph-mon[129405]: slow request 376.824705 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:20:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: 1522 slow requests, 5 included below; oldest blocked for > 377.163020 secs 2023-07-14T11:20:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: slow request 376.851954 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:20:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: slow request 376.841906 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:20:18.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: slow request 376.836164 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:20:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: slow request 376.830389 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:20:18.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:18 smithi040 ceph-mon[97954]: slow request 376.824705 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:20:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: 1522 slow requests, 5 included below; oldest blocked for > 377.163020 secs 2023-07-14T11:20:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: slow request 376.851954 seconds old, received at 2023-07-14T11:14:01.244622+0000: peer_request:mds.1:1829 currently dispatched 2023-07-14T11:20:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: slow request 376.841906 seconds old, received at 2023-07-14T11:14:01.254669+0000: peer_request:mds.1:1861 currently dispatched 2023-07-14T11:20:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: slow request 376.836164 seconds old, received at 2023-07-14T11:14:01.260412+0000: peer_request:mds.1:1893 currently dispatched 2023-07-14T11:20:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: slow request 376.830389 seconds old, received at 2023-07-14T11:14:01.266187+0000: peer_request:mds.1:1925 currently dispatched 2023-07-14T11:20:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:18 smithi045 ceph-mon[128791]: slow request 376.824705 seconds old, received at 2023-07-14T11:14:01.271871+0000: peer_request:mds.1:1957 currently dispatched 2023-07-14T11:20:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:19.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:19.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:19 smithi045 ceph-mon[128791]: pgmap v252: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:20:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:19 smithi130 ceph-mon[129405]: pgmap v252: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:20:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:19 smithi040 ceph-mon[97954]: pgmap v252: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:20:21.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:21.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:21.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:21 smithi045 ceph-mon[128791]: pgmap v253: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:21 smithi130 ceph-mon[129405]: pgmap v253: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:21 smithi040 ceph-mon[97954]: pgmap v253: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:22.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: 1514 slow requests, 5 included below; oldest blocked for > 383.164971 secs 2023-07-14T11:20:22.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: slow request 381.805343 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:22.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: slow request 381.779177 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:22.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: slow request 381.753531 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:22.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: slow request 381.725185 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:22.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:22 smithi045 ceph-mon[128791]: slow request 381.698128 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: 1514 slow requests, 5 included below; oldest blocked for > 383.164971 secs 2023-07-14T11:20:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: slow request 381.805343 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: slow request 381.779177 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: slow request 381.753531 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: slow request 381.725185 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:22 smithi130 ceph-mon[129405]: slow request 381.698128 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: 1514 slow requests, 5 included below; oldest blocked for > 383.164971 secs 2023-07-14T11:20:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: slow request 381.805343 seconds old, received at 2023-07-14T11:13:59.462207+0000: client_request(mds.1:1972 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: slow request 381.779177 seconds old, received at 2023-07-14T11:13:59.488373+0000: client_request(mds.1:2004 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: slow request 381.753531 seconds old, received at 2023-07-14T11:13:59.514019+0000: client_request(mds.1:2036 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: slow request 381.725185 seconds old, received at 2023-07-14T11:13:59.542365+0000: client_request(mds.1:2068 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:23.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:22 smithi040 ceph-mon[97954]: slow request 381.698128 seconds old, received at 2023-07-14T11:13:59.569422+0000: client_request(mds.1:2100 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: pgmap v254: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: 1509 slow requests, 5 included below; oldest blocked for > 382.163089 secs 2023-07-14T11:20:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: slow request 381.818950 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:20:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: slow request 381.813112 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:20:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: slow request 381.807476 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:20:24.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: slow request 381.801773 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:20:24.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:23 smithi130 ceph-mon[129405]: slow request 381.796057 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:20:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: pgmap v254: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: 1509 slow requests, 5 included below; oldest blocked for > 382.163089 secs 2023-07-14T11:20:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: slow request 381.818950 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:20:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: slow request 381.813112 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:20:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: slow request 381.807476 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:20:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: slow request 381.801773 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:20:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:23 smithi040 ceph-mon[97954]: slow request 381.796057 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:20:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: pgmap v254: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:20:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: 1509 slow requests, 5 included below; oldest blocked for > 382.163089 secs 2023-07-14T11:20:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: slow request 381.818950 seconds old, received at 2023-07-14T11:14:01.277696+0000: peer_request:mds.1:1989 currently dispatched 2023-07-14T11:20:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: slow request 381.813112 seconds old, received at 2023-07-14T11:14:01.283533+0000: peer_request:mds.1:2021 currently dispatched 2023-07-14T11:20:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: slow request 381.807476 seconds old, received at 2023-07-14T11:14:01.289169+0000: peer_request:mds.1:2053 currently dispatched 2023-07-14T11:20:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: slow request 381.801773 seconds old, received at 2023-07-14T11:14:01.294872+0000: peer_request:mds.1:2085 currently dispatched 2023-07-14T11:20:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:23 smithi045 ceph-mon[128791]: slow request 381.796057 seconds old, received at 2023-07-14T11:14:01.300588+0000: peer_request:mds.1:2117 currently dispatched 2023-07-14T11:20:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:25 smithi130 ceph-mon[129405]: pgmap v255: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:20:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:25 smithi040 ceph-mon[97954]: pgmap v255: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:20:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:25 smithi045 ceph-mon[128791]: pgmap v255: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:20:26.188 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:20:27.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 -- 172.21.15.40:0/2574399335 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 msgr2=0x7f3d6c072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2574399335 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c072c80 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c009990 tx=0x7f3d5c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 -- 172.21.15.40:0/2574399335 shutdown_connections 2023-07-14T11:20:27.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2574399335 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c10e700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2574399335 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c072c80 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2574399335 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 0x7f3d6c10bde0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.884 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.880+0000 7f3d709ff700 1 -- 172.21.15.40:0/2574399335 >> 172.21.15.40:0/2574399335 conn(0x7f3d6c0fb5e0 msgr2=0x7f3d6c0fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.881+0000 7f3d709ff700 1 -- 172.21.15.40:0/2574399335 shutdown_connections 2023-07-14T11:20:27.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.881+0000 7f3d709ff700 1 -- 172.21.15.40:0/2574399335 wait complete. 2023-07-14T11:20:27.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 Processor -- start 2023-07-14T11:20:27.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 -- start start 2023-07-14T11:20:27.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 0x7f3d6c198370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c19c7d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d6c19ce10 con 0x7f3d6c0731c0 2023-07-14T11:20:27.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d6c19cf80 con 0x7f3d6c0728b0 2023-07-14T11:20:27.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d709ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d6c19d140 con 0x7f3d6c10c320 2023-07-14T11:20:27.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d6a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d6a59c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49602/0 (socket says 172.21.15.40:49602) 2023-07-14T11:20:27.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.882+0000 7f3d6a59c700 1 -- 172.21.15.40:0/2939026778 learned_addr learned my addr 172.21.15.40:0/2939026778 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:27.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c19c7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 0x7f3d6c198370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 -- 172.21.15.40:0/2939026778 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 msgr2=0x7f3d6c19c7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c19c7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 -- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 msgr2=0x7f3d6c198370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 0x7f3d6c198370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 -- 172.21.15.40:0/2939026778 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d64009c40 con 0x7f3d6c0728b0 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c19c7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.883+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c005910 tx=0x7f3d5c005720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.884+0000 7f3d5b7fe700 1 -- 172.21.15.40:0/2939026778 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d5c01d070 con 0x7f3d6c0728b0 2023-07-14T11:20:27.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.884+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d5c0095f0 con 0x7f3d6c0728b0 2023-07-14T11:20:27.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.884+0000 7f3d5b7fe700 1 -- 172.21.15.40:0/2939026778 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d5c022470 con 0x7f3d6c0728b0 2023-07-14T11:20:27.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.884+0000 7f3d5b7fe700 1 -- 172.21.15.40:0/2939026778 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d5c017650 con 0x7f3d6c0728b0 2023-07-14T11:20:27.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.884+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d6c1a4a80 con 0x7f3d6c0728b0 2023-07-14T11:20:27.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.888+0000 7f3d84341b80 1 Processor -- start 2023-07-14T11:20:27.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 -- start start 2023-07-14T11:20:27.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d5b7fe700 1 -- 172.21.15.40:0/2939026778 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3d5c0225e0 con 0x7f3d6c0728b0 2023-07-14T11:20:27.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117f3ad0 0x55d1117f5eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1116eecb0 con 0x55d1117e8be0 2023-07-14T11:20:27.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d111741a40 con 0x55d1117e82d0 2023-07-14T11:20:27.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d84341b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d11173aed0 con 0x55d1117f3ad0 2023-07-14T11:20:27.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49614/0 (socket says 172.21.15.40:49614) 2023-07-14T11:20:27.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/924724423 learned_addr learned my addr 172.21.15.40:0/924724423 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:27.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117f3ad0 0x55d1117f5eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.889+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/924724423 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117f3ad0 msgr2=0x55d1117f5eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117f3ad0 0x55d1117f5eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/924724423 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 msgr2=0x55d1117f3590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/924724423 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d111568de0 con 0x55d1117e82d0 2023-07-14T11:20:27.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3d64007610 tx=0x7f3d64009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=d5465d812833758b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d58ff9700 1 -- 172.21.15.40:0/924724423 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d64009100 con 0x55d1117e82d0 2023-07-14T11:20:27.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d58ff9700 1 -- 172.21.15.40:0/924724423 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d6400f040 con 0x55d1117e82d0 2023-07-14T11:20:27.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.890+0000 7f3d58ff9700 1 -- 172.21.15.40:0/924724423 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d64017dd0 con 0x55d1117e82d0 2023-07-14T11:20:27.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d5b7fe700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 0x7f3d540a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d5b7fe700 1 -- 172.21.15.40:0/2939026778 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3d5c0ce410 con 0x7f3d6c0728b0 2023-07-14T11:20:27.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 -- 172.21.15.40:0/924724423 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 msgr2=0x55d1117e86a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7f3d64007610 tx=0x7f3d64009920 comp rx=0 tx=0).stop 2023-07-14T11:20:27.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 0x7f3d540a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 -- 172.21.15.40:0/924724423 shutdown_connections 2023-07-14T11:20:27.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117f3ad0 0x55d1117f5eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117e82d0 0x55d1117e86a0 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 --2- 172.21.15.40:0/924724423 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.891+0000 7f3d84341b80 1 -- 172.21.15.40:0/924724423 >> 172.21.15.40:0/924724423 conn(0x55d1117deda0 msgr2=0x55d1117e46a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.892+0000 7f3d84341b80 1 -- 172.21.15.40:0/924724423 shutdown_connections 2023-07-14T11:20:27.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.892+0000 7f3d84341b80 1 -- 172.21.15.40:0/924724423 wait complete. 2023-07-14T11:20:27.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.892+0000 7f3d6a59c700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 0x7f3d540a3770 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c097680 tx=0x7f3d5c022f90 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.892+0000 7f3d84341b80 1 -- start start 2023-07-14T11:20:27.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.893+0000 7f3d84341b80 1 Processor -- start 2023-07-14T11:20:27.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.893+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f0760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117f0d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1116daac0 con 0x55d1117e8be0 2023-07-14T11:20:27.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1116da7c0 con 0x55d1117f3ad0 2023-07-14T11:20:27.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1115be920 con 0x55d1117e82d0 2023-07-14T11:20:27.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d84341b80 10 client.0 _ll_register_callbacks cb 0x55d1116d8760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:20:27.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44306/0 (socket says 172.21.15.40:44306) 2023-07-14T11:20:27.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/56861014 learned_addr learned my addr 172.21.15.40:0/56861014 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:27.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117f0d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f0760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 msgr2=0x55d1117f0d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117f0d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/56861014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 msgr2=0x55d1117f0760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f0760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.894+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/56861014 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d64009260 con 0x55d1117e82d0 2023-07-14T11:20:27.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f3d640075e0 tx=0x7f3d64017930 comp rx=0 tx=0).ready entity=mon.1 client_cookie=57a88d3b96ffa636 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d51ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:20:27.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d64004ad0 con 0x55d1117e82d0 2023-07-14T11:20:27.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d1116944f0 con 0x55d1117e82d0 2023-07-14T11:20:27.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3d64004c30 con 0x55d1117e82d0 2023-07-14T11:20:27.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=133) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3d64004df0 con 0x55d1117e82d0 2023-07-14T11:20:27.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d84341b80 10 client.26392 fetch_fsmap learned FSMap version 133 2023-07-14T11:20:27.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.895+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=133,monmap=4+}) v3 -- 0x55d111551c60 con 0x55d1117e82d0 2023-07-14T11:20:27.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d1116944f0 con 0x55d1117e82d0 2023-07-14T11:20:27.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 133) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3d640053c0 con 0x55d1117e82d0 2023-07-14T11:20:27.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 10 client.26392 fetch_fsmap finished waiting for FSMap version 133 2023-07-14T11:20:27.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 10 client.26392 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:20:27.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 20 client.26392 populate_metadata read hostname 'smithi040' 2023-07-14T11:20:27.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d1118033e0 0x55d1118057f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 4 client.26392 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:20:27.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d111733eb0 con 0x55d1118033e0 2023-07-14T11:20:27.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.896+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d1118033e0 0x55d1118057f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.897+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d1118033e0 0x55d1118057f0 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=f7fba20172a76970 server_cookie=f5dc257738d2c95f in_seq=0 out_seq=0 2023-07-14T11:20:27.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.897+0000 7f3d51ffb700 10 client.26392 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:27.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.897+0000 7f3d51ffb700 1 -- 172.21.15.40:0/56861014 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55d111733eb0 con 0x55d1118033e0 2023-07-14T11:20:27.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.897+0000 7f3d51ffb700 10 client.26392 handle_command_reply: tid=0 2023-07-14T11:20:27.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 client.26392 shutdown 2023-07-14T11:20:27.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 msgr2=0x55d1117f34e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 crypto rx=0x7f3d640075e0 tx=0x7f3d64017930 comp rx=0 tx=0).stop 2023-07-14T11:20:27.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 shutdown_connections 2023-07-14T11:20:27.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d1118033e0 0x55d1118057f0 crc :-1 s=READY pgs=103 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117f0d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d51ffb700 0 client.26392 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:27.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117f34e0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 --2- 172.21.15.40:0/56861014 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f0760 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 >> 172.21.15.40:0/56861014 conn(0x55d1117deda0 msgr2=0x55d1117e4230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 shutdown_connections 2023-07-14T11:20:27.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 1 -- 172.21.15.40:0/56861014 wait complete. 2023-07-14T11:20:27.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.898+0000 7f3d84341b80 20 client.26392 trim_cache size 0 max 16384 2023-07-14T11:20:27.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.933+0000 7f3d84341b80 1 Processor -- start 2023-07-14T11:20:27.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 -- start start 2023-07-14T11:20:27.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.937 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117eba90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117e9e30 con 0x55d1117e8be0 2023-07-14T11:20:27.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117f1bf0 con 0x55d1117f3ad0 2023-07-14T11:20:27.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d84341b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117e9570 con 0x55d1117e82d0 2023-07-14T11:20:27.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d6ad9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44312/0 (socket says 172.21.15.40:44312) 2023-07-14T11:20:27.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d69d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/3892244205 learned_addr learned my addr 172.21.15.40:0/3892244205 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:27.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.934+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117eba90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/3892244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 msgr2=0x55d1117eba90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117eba90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/3892244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 msgr2=0x55d1117f3010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 -- 172.21.15.40:0/3892244205 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d11158f250 con 0x55d1117e82d0 2023-07-14T11:20:27.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f3d64009230 tx=0x7f3d640099a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=609747a95567dd2c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d597fa700 1 -- 172.21.15.40:0/3892244205 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d64017460 con 0x55d1117e82d0 2023-07-14T11:20:27.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d597fa700 1 -- 172.21.15.40:0/3892244205 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d64005530 con 0x55d1117e82d0 2023-07-14T11:20:27.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.935+0000 7f3d597fa700 1 -- 172.21.15.40:0/3892244205 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d6400eea0 con 0x55d1117e82d0 2023-07-14T11:20:27.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 -- 172.21.15.40:0/3892244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 msgr2=0x55d1117fa4a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 crypto rx=0x7f3d64009230 tx=0x7f3d640099a0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 -- 172.21.15.40:0/3892244205 shutdown_connections 2023-07-14T11:20:27.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117eba90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e82d0 0x55d1117fa4a0 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3892244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e8be0 0x55d1117f3010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 -- 172.21.15.40:0/3892244205 >> 172.21.15.40:0/3892244205 conn(0x55d1117deda0 msgr2=0x55d1117fec50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 -- 172.21.15.40:0/3892244205 shutdown_connections 2023-07-14T11:20:27.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.936+0000 7f3d84341b80 1 -- 172.21.15.40:0/3892244205 wait complete. 2023-07-14T11:20:27.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 -- start start 2023-07-14T11:20:27.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 Processor -- start 2023-07-14T11:20:27.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 0x55d1117ee050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117ecf20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117ee630 con 0x55d1117e82d0 2023-07-14T11:20:27.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117ee7a0 con 0x55d1117f3ad0 2023-07-14T11:20:27.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d1117ed460 con 0x55d1117e8be0 2023-07-14T11:20:27.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.937+0000 7f3d84341b80 10 client.0 _ll_register_callbacks cb 0x7f3d64017e80 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:20:27.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44322/0 (socket says 172.21.15.40:44322) 2023-07-14T11:20:27.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 -- 172.21.15.40:0/3202093120 learned_addr learned my addr 172.21.15.40:0/3202093120 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:27.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 0x55d1117ee050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117ecf20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 -- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 msgr2=0x55d1117ecf20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117ecf20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 -- 172.21.15.40:0/3202093120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 msgr2=0x55d1117ee050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:27.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 0x55d1117ee050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 -- 172.21.15.40:0/3202093120 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d111551c60 con 0x55d1117e8be0 2023-07-14T11:20:27.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 0x55d1117ee050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:20:27.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d69d9b700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x55d1117f1da0 tx=0x7f3d6001fe20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9f9cccc0e01c620e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:27.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.938+0000 7f3d51ffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:20:27.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d60016070 con 0x55d1117e8be0 2023-07-14T11:20:27.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3d6400f040 con 0x55d1117e8be0 2023-07-14T11:20:27.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3d6001f240 con 0x55d1117e8be0 2023-07-14T11:20:27.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=133) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3d6001f420 con 0x55d1117e8be0 2023-07-14T11:20:27.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 10 client.26398 fetch_fsmap learned FSMap version 133 2023-07-14T11:20:27.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=133,monmap=4+}) v3 -- 0x55d1115672b0 con 0x55d1117e8be0 2023-07-14T11:20:27.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d6400f040 con 0x55d1117e8be0 2023-07-14T11:20:27.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 133) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3d60010730 con 0x55d1117e8be0 2023-07-14T11:20:27.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 10 client.26398 fetch_fsmap finished waiting for FSMap version 133 2023-07-14T11:20:27.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 10 client.26398 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:20:27.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 20 client.26398 populate_metadata read hostname 'smithi040' 2023-07-14T11:20:27.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d111858410 0x55d11185a820 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:27.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 4 client.26398 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:20:27.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.939+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d1116f6050 con 0x55d111858410 2023-07-14T11:20:27.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.940+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d111858410 0x55d11185a820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:27.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.940+0000 7f3d6ad9d700 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d111858410 0x55d11185a820 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=ce634cd2f907922f server_cookie=b6ecabccfc565915 in_seq=0 out_seq=0 2023-07-14T11:20:27.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.940+0000 7f3d51ffb700 10 client.26398 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:27.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.940+0000 7f3d51ffb700 1 -- 172.21.15.40:0/3202093120 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55d1116f6050 con 0x55d111858410 2023-07-14T11:20:27.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.940+0000 7f3d51ffb700 10 client.26398 handle_command_reply: tid=0 2023-07-14T11:20:27.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 client.26398 shutdown 2023-07-14T11:20:27.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 msgr2=0x55d1117ec9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x55d1117f1da0 tx=0x7f3d6001fe20 comp rx=0 tx=0).stop 2023-07-14T11:20:27.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 shutdown_connections 2023-07-14T11:20:27.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d111858410 0x55d11185a820 crc :-1 s=READY pgs=104 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d1117f3ad0 0x55d1117ecf20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d1117e82d0 0x55d1117ee050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d51ffb700 0 client.26398 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:27.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 --2- 172.21.15.40:0/3202093120 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d1117e8be0 0x55d1117ec9e0 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.955 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 >> 172.21.15.40:0/3202093120 conn(0x55d1117deda0 msgr2=0x55d1117eba90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 shutdown_connections 2023-07-14T11:20:27.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 1 -- 172.21.15.40:0/3202093120 wait complete. 2023-07-14T11:20:27.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.941+0000 7f3d84341b80 20 client.26398 trim_cache size 0 max 16384 2023-07-14T11:20:27.957 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:20:27.957 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:20:27.957 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:20:27.957 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:20:27.957 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:20:27.958 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:20:27.958 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:20:27.958 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:20:27.958 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:20:27.959 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:20:27.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 msgr2=0x7f3d540a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 0x7f3d540a3770 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c097680 tx=0x7f3d5c022f90 comp rx=0 tx=0).stop 2023-07-14T11:20:27.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 msgr2=0x7f3d6c197e30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:27.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c005910 tx=0x7f3d5c005720 comp rx=0 tx=0).stop 2023-07-14T11:20:27.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 shutdown_connections 2023-07-14T11:20:27.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d540a12c0 0x7f3d540a3770 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d6c10c320 0x7f3d6c19c7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d6c0728b0 0x7f3d6c197e30 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 --2- 172.21.15.40:0/2939026778 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d6c0731c0 0x7f3d6c198370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:27.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 >> 172.21.15.40:0/2939026778 conn(0x7f3d6c0fb5e0 msgr2=0x7f3d6c10e490 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:27.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 shutdown_connections 2023-07-14T11:20:27.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:27.942+0000 7f3d709ff700 1 -- 172.21.15.40:0/2939026778 wait complete. 2023-07-14T11:20:27.963 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:20:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: 1500 slow requests, 5 included below; oldest blocked for > 388.409631 secs 2023-07-14T11:20:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: slow request 386.916241 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: slow request 386.883924 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: slow request 386.858206 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: slow request 386.832934 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: slow request 386.807880 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:27 smithi130 ceph-mon[129405]: pgmap v256: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: 1500 slow requests, 5 included below; oldest blocked for > 388.409631 secs 2023-07-14T11:20:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: slow request 386.916241 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: slow request 386.883924 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: slow request 386.858206 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: slow request 386.832934 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: slow request 386.807880 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:27 smithi040 ceph-mon[97954]: pgmap v256: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: 1500 slow requests, 5 included below; oldest blocked for > 388.409631 secs 2023-07-14T11:20:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: slow request 386.916241 seconds old, received at 2023-07-14T11:13:59.595969+0000: client_request(mds.1:2132 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: slow request 386.883924 seconds old, received at 2023-07-14T11:13:59.628287+0000: client_request(mds.1:2164 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: slow request 386.858206 seconds old, received at 2023-07-14T11:13:59.654005+0000: client_request(mds.1:2196 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: slow request 386.832934 seconds old, received at 2023-07-14T11:13:59.679277+0000: client_request(mds.1:2228 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: slow request 386.807880 seconds old, received at 2023-07-14T11:13:59.704330+0000: client_request(mds.1:2260 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:28.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:27 smithi045 ceph-mon[128791]: pgmap v256: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: 1496 slow requests, 5 included below; oldest blocked for > 387.163194 secs 2023-07-14T11:20:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: slow request 386.790475 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:20:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: slow request 386.784747 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:20:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: slow request 386.779064 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:20:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: slow request 386.773311 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:20:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:28 smithi130 ceph-mon[129405]: slow request 386.767318 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:20:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: 1496 slow requests, 5 included below; oldest blocked for > 387.163194 secs 2023-07-14T11:20:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: slow request 386.790475 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:20:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: slow request 386.784747 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:20:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: slow request 386.779064 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:20:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: slow request 386.773311 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:20:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:28 smithi040 ceph-mon[97954]: slow request 386.767318 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:20:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: 1496 slow requests, 5 included below; oldest blocked for > 387.163194 secs 2023-07-14T11:20:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: slow request 386.790475 seconds old, received at 2023-07-14T11:14:01.306274+0000: peer_request:mds.1:2149 currently dispatched 2023-07-14T11:20:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: slow request 386.784747 seconds old, received at 2023-07-14T11:14:01.312003+0000: peer_request:mds.1:2181 currently dispatched 2023-07-14T11:20:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: slow request 386.779064 seconds old, received at 2023-07-14T11:14:01.317685+0000: peer_request:mds.1:2213 currently dispatched 2023-07-14T11:20:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: slow request 386.773311 seconds old, received at 2023-07-14T11:14:01.323439+0000: peer_request:mds.1:2245 currently dispatched 2023-07-14T11:20:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:28 smithi045 ceph-mon[128791]: slow request 386.767318 seconds old, received at 2023-07-14T11:14:01.329431+0000: peer_request:mds.1:2277 currently dispatched 2023-07-14T11:20:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:29 smithi130 ceph-mon[129405]: pgmap v257: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:29 smithi040 ceph-mon[97954]: pgmap v257: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:29 smithi045 ceph-mon[128791]: pgmap v257: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:31 smithi130 ceph-mon[129405]: pgmap v258: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:31 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:20:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:31 smithi040 ceph-mon[97954]: pgmap v258: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:31 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:20:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:31 smithi045 ceph-mon[128791]: pgmap v258: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:31 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:20:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: 1488 slow requests, 5 included below; oldest blocked for > 393.449069 secs 2023-07-14T11:20:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: slow request 391.819760 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: slow request 391.780570 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: slow request 391.749385 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: slow request 391.723144 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: slow request 391.696745 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:33.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:32 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: 1488 slow requests, 5 included below; oldest blocked for > 393.449069 secs 2023-07-14T11:20:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: slow request 391.819760 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: slow request 391.780570 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: slow request 391.749385 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: slow request 391.723144 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: slow request 391.696745 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:33.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:32 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: 1488 slow requests, 5 included below; oldest blocked for > 393.449069 secs 2023-07-14T11:20:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: slow request 391.819760 seconds old, received at 2023-07-14T11:13:59.731889+0000: client_request(mds.1:2292 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: slow request 391.780570 seconds old, received at 2023-07-14T11:13:59.771079+0000: client_request(mds.1:2324 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: slow request 391.749385 seconds old, received at 2023-07-14T11:13:59.802264+0000: client_request(mds.1:2356 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: slow request 391.723144 seconds old, received at 2023-07-14T11:13:59.828505+0000: client_request(mds.1:2388 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: slow request 391.696745 seconds old, received at 2023-07-14T11:13:59.854904+0000: client_request(mds.1:2420 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:33.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:32 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: pgmap v259: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: 1483 slow requests, 5 included below; oldest blocked for > 392.163312 secs 2023-07-14T11:20:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: slow request 391.761377 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:20:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: slow request 391.755477 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:20:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: slow request 391.749535 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:20:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: slow request 391.743578 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:20:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:33 smithi130 ceph-mon[129405]: slow request 391.737700 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:20:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: pgmap v259: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: 1483 slow requests, 5 included below; oldest blocked for > 392.163312 secs 2023-07-14T11:20:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: slow request 391.761377 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:20:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: slow request 391.755477 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:20:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: slow request 391.749535 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:20:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: slow request 391.743578 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:20:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:33 smithi040 ceph-mon[97954]: slow request 391.737700 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:20:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: pgmap v259: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: 1483 slow requests, 5 included below; oldest blocked for > 392.163312 secs 2023-07-14T11:20:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: slow request 391.761377 seconds old, received at 2023-07-14T11:14:01.335490+0000: peer_request:mds.1:2309 currently dispatched 2023-07-14T11:20:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: slow request 391.755477 seconds old, received at 2023-07-14T11:14:01.341390+0000: peer_request:mds.1:2341 currently dispatched 2023-07-14T11:20:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: slow request 391.749535 seconds old, received at 2023-07-14T11:14:01.347333+0000: peer_request:mds.1:2373 currently dispatched 2023-07-14T11:20:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: slow request 391.743578 seconds old, received at 2023-07-14T11:14:01.353290+0000: peer_request:mds.1:2405 currently dispatched 2023-07-14T11:20:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:33 smithi045 ceph-mon[128791]: slow request 391.737700 seconds old, received at 2023-07-14T11:14:01.359168+0000: peer_request:mds.1:2437 currently dispatched 2023-07-14T11:20:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: pgmap v260: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:20:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:20:36.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:35 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:20:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: pgmap v260: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:20:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:20:36.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:35 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:20:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: pgmap v260: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:36.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:20:36.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:20:36.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:35 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:20:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:37.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:37.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: 1475 slow requests, 5 included below; oldest blocked for > 398.549438 secs 2023-07-14T11:20:37.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: slow request 396.771101 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: slow request 396.744995 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: slow request 396.718351 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.797 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: slow request 396.689845 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: slow request 396.662331 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.798 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:37 smithi040 ceph-mon[97954]: pgmap v261: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:37.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: 1475 slow requests, 5 included below; oldest blocked for > 398.549438 secs 2023-07-14T11:20:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: slow request 396.771101 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: slow request 396.744995 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: slow request 396.718351 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: slow request 396.689845 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: slow request 396.662331 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:37.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:37 smithi045 ceph-mon[128791]: pgmap v261: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: 1475 slow requests, 5 included below; oldest blocked for > 398.549438 secs 2023-07-14T11:20:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: slow request 396.771101 seconds old, received at 2023-07-14T11:13:59.880917+0000: client_request(mds.1:2452 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: slow request 396.744995 seconds old, received at 2023-07-14T11:13:59.907023+0000: client_request(mds.1:2484 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: slow request 396.718351 seconds old, received at 2023-07-14T11:13:59.933667+0000: client_request(mds.1:2516 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: slow request 396.689845 seconds old, received at 2023-07-14T11:13:59.962173+0000: client_request(mds.1:2548 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:38.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: slow request 396.662331 seconds old, received at 2023-07-14T11:13:59.989687+0000: client_request(mds.1:2580 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:38.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:37 smithi130 ceph-mon[129405]: pgmap v261: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: 1471 slow requests, 5 included below; oldest blocked for > 397.163373 secs 2023-07-14T11:20:38.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: slow request 396.731900 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:20:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: slow request 396.726023 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:20:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: slow request 396.720143 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:20:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: slow request 396.714221 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:20:38.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:38 smithi045 ceph-mon[128791]: slow request 396.708365 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:20:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: 1471 slow requests, 5 included below; oldest blocked for > 397.163373 secs 2023-07-14T11:20:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: slow request 396.731900 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:20:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: slow request 396.726023 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:20:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: slow request 396.720143 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:20:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: slow request 396.714221 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:20:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:38 smithi130 ceph-mon[129405]: slow request 396.708365 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:20:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: 1471 slow requests, 5 included below; oldest blocked for > 397.163373 secs 2023-07-14T11:20:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: slow request 396.731900 seconds old, received at 2023-07-14T11:14:01.365029+0000: peer_request:mds.1:2469 currently dispatched 2023-07-14T11:20:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: slow request 396.726023 seconds old, received at 2023-07-14T11:14:01.370906+0000: peer_request:mds.1:2501 currently dispatched 2023-07-14T11:20:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: slow request 396.720143 seconds old, received at 2023-07-14T11:14:01.376786+0000: peer_request:mds.1:2533 currently dispatched 2023-07-14T11:20:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: slow request 396.714221 seconds old, received at 2023-07-14T11:14:01.382708+0000: peer_request:mds.1:2565 currently dispatched 2023-07-14T11:20:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:38 smithi040 ceph-mon[97954]: slow request 396.708365 seconds old, received at 2023-07-14T11:14:01.388564+0000: peer_request:mds.1:2597 currently dispatched 2023-07-14T11:20:39.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:39.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:39.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:39 smithi045 ceph-mon[128791]: pgmap v262: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:39 smithi130 ceph-mon[129405]: pgmap v262: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:39 smithi040 ceph-mon[97954]: pgmap v262: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:41 smithi045 ceph-mon[128791]: pgmap v263: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:41 smithi130 ceph-mon[129405]: pgmap v263: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:41 smithi040 ceph-mon[97954]: pgmap v263: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:43.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: 1462 slow requests, 5 included below; oldest blocked for > 403.763134 secs 2023-07-14T11:20:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: slow request 401.848369 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: slow request 401.822707 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: slow request 401.796056 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: slow request 401.769303 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:42 smithi130 ceph-mon[129405]: slow request 401.743156 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: 1462 slow requests, 5 included below; oldest blocked for > 403.763134 secs 2023-07-14T11:20:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: slow request 401.848369 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: slow request 401.822707 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: slow request 401.796056 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: slow request 401.769303 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:42 smithi040 ceph-mon[97954]: slow request 401.743156 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: 1462 slow requests, 5 included below; oldest blocked for > 403.763134 secs 2023-07-14T11:20:43.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: slow request 401.848369 seconds old, received at 2023-07-14T11:14:00.017344+0000: client_request(mds.1:2612 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: slow request 401.822707 seconds old, received at 2023-07-14T11:14:00.043007+0000: client_request(mds.1:2644 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: slow request 401.796056 seconds old, received at 2023-07-14T11:14:00.069657+0000: client_request(mds.1:2676 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: slow request 401.769303 seconds old, received at 2023-07-14T11:14:00.096410+0000: client_request(mds.1:2708 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:43.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:42 smithi045 ceph-mon[128791]: slow request 401.743156 seconds old, received at 2023-07-14T11:14:00.122557+0000: client_request(mds.1:2740 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: pgmap v264: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: 1458 slow requests, 5 included below; oldest blocked for > 402.163507 secs 2023-07-14T11:20:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: slow request 401.702432 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:20:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: slow request 401.696529 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:20:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: slow request 401.690764 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:20:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: slow request 401.684822 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:20:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:43 smithi130 ceph-mon[129405]: slow request 401.678964 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:20:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: pgmap v264: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: 1458 slow requests, 5 included below; oldest blocked for > 402.163507 secs 2023-07-14T11:20:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: slow request 401.702432 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:20:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: slow request 401.696529 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:20:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: slow request 401.690764 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:20:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: slow request 401.684822 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:20:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:43 smithi040 ceph-mon[97954]: slow request 401.678964 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:20:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: pgmap v264: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: 1458 slow requests, 5 included below; oldest blocked for > 402.163507 secs 2023-07-14T11:20:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: slow request 401.702432 seconds old, received at 2023-07-14T11:14:01.394632+0000: peer_request:mds.1:2629 currently dispatched 2023-07-14T11:20:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: slow request 401.696529 seconds old, received at 2023-07-14T11:14:01.400535+0000: peer_request:mds.1:2661 currently dispatched 2023-07-14T11:20:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: slow request 401.690764 seconds old, received at 2023-07-14T11:14:01.406300+0000: peer_request:mds.1:2693 currently dispatched 2023-07-14T11:20:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: slow request 401.684822 seconds old, received at 2023-07-14T11:14:01.412241+0000: peer_request:mds.1:2725 currently dispatched 2023-07-14T11:20:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:43 smithi045 ceph-mon[128791]: slow request 401.678964 seconds old, received at 2023-07-14T11:14:01.418099+0000: peer_request:mds.1:2757 currently dispatched 2023-07-14T11:20:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:45 smithi130 ceph-mon[129405]: pgmap v265: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:45 smithi040 ceph-mon[97954]: pgmap v265: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:45 smithi045 ceph-mon[128791]: pgmap v265: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: pgmap v266: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: 1449 slow requests, 5 included below; oldest blocked for > 408.839935 secs 2023-07-14T11:20:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: slow request 406.793440 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: slow request 406.767415 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: slow request 406.741136 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: slow request 406.713593 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:47 smithi130 ceph-mon[129405]: slow request 408.009721 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: pgmap v266: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: 1449 slow requests, 5 included below; oldest blocked for > 408.839935 secs 2023-07-14T11:20:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: slow request 406.793440 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: slow request 406.767415 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: slow request 406.741136 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: slow request 406.713593 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:47 smithi040 ceph-mon[97954]: slow request 408.009721 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: pgmap v266: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: 1449 slow requests, 5 included below; oldest blocked for > 408.839935 secs 2023-07-14T11:20:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: slow request 406.793440 seconds old, received at 2023-07-14T11:14:00.149075+0000: client_request(mds.1:2772 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: slow request 406.767415 seconds old, received at 2023-07-14T11:14:00.175100+0000: client_request(mds.1:2804 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: slow request 406.741136 seconds old, received at 2023-07-14T11:14:00.201379+0000: client_request(mds.1:2836 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: slow request 406.713593 seconds old, received at 2023-07-14T11:14:00.228922+0000: client_request(mds.1:2868 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:48.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:47 smithi045 ceph-mon[128791]: slow request 408.009721 seconds old, received at 2023-07-14T11:13:58.932794+0000: client_request(mds.1:1461 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: 1445 slow requests, 5 included below; oldest blocked for > 407.163616 secs 2023-07-14T11:20:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: slow request 406.673175 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:20:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: slow request 406.667281 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:20:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: slow request 406.661444 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:20:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: slow request 406.970402 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:20:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:48 smithi130 ceph-mon[129405]: slow request 406.960418 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:20:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: 1445 slow requests, 5 included below; oldest blocked for > 407.163616 secs 2023-07-14T11:20:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: slow request 406.673175 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:20:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: slow request 406.667281 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:20:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: slow request 406.661444 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:20:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: slow request 406.970402 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:20:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:48 smithi040 ceph-mon[97954]: slow request 406.960418 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:20:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: 1445 slow requests, 5 included below; oldest blocked for > 407.163616 secs 2023-07-14T11:20:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: slow request 406.673175 seconds old, received at 2023-07-14T11:14:01.423997+0000: peer_request:mds.1:2789 currently dispatched 2023-07-14T11:20:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: slow request 406.667281 seconds old, received at 2023-07-14T11:14:01.429891+0000: peer_request:mds.1:2821 currently dispatched 2023-07-14T11:20:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: slow request 406.661444 seconds old, received at 2023-07-14T11:14:01.435728+0000: peer_request:mds.1:2853 currently dispatched 2023-07-14T11:20:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: slow request 406.970402 seconds old, received at 2023-07-14T11:14:01.126770+0000: peer_request:mds.1:1446 currently dispatched 2023-07-14T11:20:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:48 smithi045 ceph-mon[128791]: slow request 406.960418 seconds old, received at 2023-07-14T11:14:01.136754+0000: peer_request:mds.1:1478 currently dispatched 2023-07-14T11:20:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:49 smithi130 ceph-mon[129405]: pgmap v267: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:49 smithi040 ceph-mon[97954]: pgmap v267: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:49 smithi045 ceph-mon[128791]: pgmap v267: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:51 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:51 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:52.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:51 smithi130 ceph-mon[129405]: pgmap v268: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:51 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:51 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:51 smithi040 ceph-mon[97954]: pgmap v268: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:51 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:51 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:51 smithi045 ceph-mon[128791]: pgmap v268: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: 1434 slow requests, 5 included below; oldest blocked for > 414.034222 secs 2023-07-14T11:20:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 413.162308 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 413.126172 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 413.092581 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 413.065551 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 413.032156 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: pgmap v269: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: 1431 slow requests, 5 included below; oldest blocked for > 412.163754 secs 2023-07-14T11:20:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 411.950480 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:20:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 411.940402 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:20:54.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 411.930393 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:20:54.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 411.922273 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:20:54.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:53 smithi130 ceph-mon[129405]: slow request 411.912464 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:20:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: 1434 slow requests, 5 included below; oldest blocked for > 414.034222 secs 2023-07-14T11:20:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 413.162308 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 413.126172 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 413.092581 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 413.065551 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 413.032156 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: pgmap v269: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: 1431 slow requests, 5 included below; oldest blocked for > 412.163754 secs 2023-07-14T11:20:54.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 411.950480 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:20:54.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 411.940402 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:20:54.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 411.930393 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:20:54.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 411.922273 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:20:54.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:53 smithi040 ceph-mon[97954]: slow request 411.912464 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:20:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: 1434 slow requests, 5 included below; oldest blocked for > 414.034222 secs 2023-07-14T11:20:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 413.162308 seconds old, received at 2023-07-14T11:13:58.974493+0000: client_request(mds.1:1493 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 413.126172 seconds old, received at 2023-07-14T11:13:59.010629+0000: client_request(mds.1:1525 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 413.092581 seconds old, received at 2023-07-14T11:13:59.044221+0000: client_request(mds.1:1557 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 413.065551 seconds old, received at 2023-07-14T11:13:59.071250+0000: client_request(mds.1:1589 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 413.032156 seconds old, received at 2023-07-14T11:13:59.104646+0000: client_request(mds.1:1621 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: pgmap v269: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: 1431 slow requests, 5 included below; oldest blocked for > 412.163754 secs 2023-07-14T11:20:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 411.950480 seconds old, received at 2023-07-14T11:14:01.146830+0000: peer_request:mds.1:1510 currently dispatched 2023-07-14T11:20:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 411.940402 seconds old, received at 2023-07-14T11:14:01.156908+0000: peer_request:mds.1:1542 currently dispatched 2023-07-14T11:20:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 411.930393 seconds old, received at 2023-07-14T11:14:01.166917+0000: peer_request:mds.1:1574 currently dispatched 2023-07-14T11:20:54.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 411.922273 seconds old, received at 2023-07-14T11:14:01.175037+0000: peer_request:mds.1:1606 currently dispatched 2023-07-14T11:20:54.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:53 smithi045 ceph-mon[128791]: slow request 411.912464 seconds old, received at 2023-07-14T11:14:01.184846+0000: peer_request:mds.1:1638 currently dispatched 2023-07-14T11:20:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:55 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:55 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:55 smithi130 ceph-mon[129405]: pgmap v270: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:55 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:55 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:55 smithi040 ceph-mon[97954]: pgmap v270: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:55 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:20:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:55 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:55 smithi045 ceph-mon[128791]: pgmap v270: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:20:57.964 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:20:58.033 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:58.034 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:58.034 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:57 smithi040 ceph-mon[97954]: pgmap v271: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:57 smithi130 ceph-mon[129405]: pgmap v271: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:20:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:20:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:57 smithi045 ceph-mon[128791]: pgmap v271: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:20:58.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 -- 172.21.15.40:0/542441991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef28105ae0 msgr2=0x7fef28105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 --2- 172.21.15.40:0/542441991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef28105ae0 0x7fef28105eb0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7fef24009990 tx=0x7fef24009ca0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 -- 172.21.15.40:0/542441991 shutdown_connections 2023-07-14T11:20:58.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 --2- 172.21.15.40:0/542441991 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28100480 0x7fef2810c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 --2- 172.21.15.40:0/542441991 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef28105ae0 0x7fef28105eb0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 --2- 172.21.15.40:0/542441991 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef280ffad0 0x7fef280fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 -- 172.21.15.40:0/542441991 >> 172.21.15.40:0/542441991 conn(0x7fef280fb5f0 msgr2=0x7fef280fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 -- 172.21.15.40:0/542441991 shutdown_connections 2023-07-14T11:20:58.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.200+0000 7fef2fbff700 1 -- 172.21.15.40:0/542441991 wait complete. 2023-07-14T11:20:58.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.202+0000 7fef2fbff700 1 Processor -- start 2023-07-14T11:20:58.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.202+0000 7fef2fbff700 1 -- start start 2023-07-14T11:20:58.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.207 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 0x7fef28072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28105ae0 0x7fef281a3da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef28073580 con 0x7fef28105ae0 2023-07-14T11:20:58.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef281a42e0 con 0x7fef28100480 2023-07-14T11:20:58.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2fbff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef281a4420 con 0x7fef280ffad0 2023-07-14T11:20:58.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52944/0 (socket says 172.21.15.40:52944) 2023-07-14T11:20:58.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 -- 172.21.15.40:0/1200016285 learned_addr learned my addr 172.21.15.40:0/1200016285 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:58.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2e19c700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28105ae0 0x7fef281a3da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 0x7fef28072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 -- 172.21.15.40:0/1200016285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 msgr2=0x7fef28072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 0x7fef28072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 -- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28105ae0 msgr2=0x7fef281a3da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28105ae0 0x7fef281a3da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d99b700 1 -- 172.21.15.40:0/1200016285 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef240095f0 con 0x7fef280ffad0 2023-07-14T11:20:58.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.203+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 0x7fef28072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:20:58.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fef24005910 tx=0x7fef240057d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef1effd700 1 -- 172.21.15.40:0/1200016285 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef2401d070 con 0x7fef280ffad0 2023-07-14T11:20:58.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef1effd700 1 -- 172.21.15.40:0/1200016285 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fef24022470 con 0x7fef280ffad0 2023-07-14T11:20:58.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef281a4620 con 0x7fef280ffad0 2023-07-14T11:20:58.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef1effd700 1 -- 172.21.15.40:0/1200016285 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef24017650 con 0x7fef280ffad0 2023-07-14T11:20:58.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.204+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef281a4aa0 con 0x7fef280ffad0 2023-07-14T11:20:58.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.208+0000 7fef1effd700 1 -- 172.21.15.40:0/1200016285 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fef240225e0 con 0x7fef280ffad0 2023-07-14T11:20:58.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.209+0000 7fef4354db80 1 Processor -- start 2023-07-14T11:20:58.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.209+0000 7fef4354db80 1 -- start start 2023-07-14T11:20:58.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.209+0000 7fef4354db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.209+0000 7fef4354db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da077590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef4354db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 0x55d6da079eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef4354db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9f72cb0 con 0x55d6da077ad0 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef4354db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9fc5a40 con 0x55d6da06c2d0 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef4354db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9fbeed0 con 0x55d6da06cbe0 2023-07-14T11:20:58.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:47432/0 (socket says 172.21.15.40:47432) 2023-07-14T11:20:58.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 -- 172.21.15.40:0/741077685 learned_addr learned my addr 172.21.15.40:0/741077685 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:58.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2d99b700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da077590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 -- 172.21.15.40:0/741077685 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 msgr2=0x55d6da077590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef1effd700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 0x7fef140a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2d19a700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 0x55d6da079eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da077590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 -- 172.21.15.40:0/741077685 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 msgr2=0x55d6da079eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 0x55d6da079eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2e19c700 1 -- 172.21.15.40:0/741077685 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d6d9decde0 con 0x55d6da06c2d0 2023-07-14T11:20:58.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2d99b700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da077590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:20:58.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef2d19a700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 0x55d6da079eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:20:58.218 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.210+0000 7fef1effd700 1 -- 172.21.15.40:0/1200016285 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fef240ce4d0 con 0x7fef280ffad0 2023-07-14T11:20:58.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.211+0000 7fef2e19c700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 0x7fef140a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.211+0000 7fef2e19c700 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fef200093e0 tx=0x7fef20009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ad4b0fe000d4f5b2 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.211+0000 7fef13fff700 1 -- 172.21.15.40:0/741077685 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef2000e070 con 0x55d6da06c2d0 2023-07-14T11:20:58.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.211+0000 7fef13fff700 1 -- 172.21.15.40:0/741077685 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fef2000f040 con 0x55d6da06c2d0 2023-07-14T11:20:58.219 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.211+0000 7fef13fff700 1 -- 172.21.15.40:0/741077685 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef20020a10 con 0x55d6da06c2d0 2023-07-14T11:20:58.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef2e19c700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 0x7fef140a37f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fef2001f000 tx=0x7fef20022000 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 -- 172.21.15.40:0/741077685 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 msgr2=0x55d6da06c6a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7fef200093e0 tx=0x7fef20009920 comp rx=0 tx=0).stop 2023-07-14T11:20:58.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 -- 172.21.15.40:0/741077685 shutdown_connections 2023-07-14T11:20:58.220 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da077ad0 0x55d6da079eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da06c2d0 0x55d6da06c6a0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 --2- 172.21.15.40:0/741077685 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da077590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 -- 172.21.15.40:0/741077685 >> 172.21.15.40:0/741077685 conn(0x55d6da062da0 msgr2=0x55d6da0686a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 -- 172.21.15.40:0/741077685 shutdown_connections 2023-07-14T11:20:58.221 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.212+0000 7fef4354db80 1 -- 172.21.15.40:0/741077685 wait complete. 2023-07-14T11:20:58.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.213+0000 7fef4354db80 1 -- start start 2023-07-14T11:20:58.222 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.213+0000 7fef4354db80 1 Processor -- start 2023-07-14T11:20:58.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da0746a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da074c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9f5eac0 con 0x55d6da06cbe0 2023-07-14T11:20:58.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9f5e7c0 con 0x55d6da077ad0 2023-07-14T11:20:58.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9e42920 con 0x55d6da06c2d0 2023-07-14T11:20:58.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef4354db80 10 client.0 _ll_register_callbacks cb 0x55d6d9f5c760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52976/0 (socket says 172.21.15.40:52976) 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef2d99b700 1 -- 172.21.15.40:0/3081131590 learned_addr learned my addr 172.21.15.40:0/3081131590 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef2d19a700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da0746a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.214+0000 7fef2e19c700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da074c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 -- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 msgr2=0x55d6da074c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da074c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 -- 172.21.15.40:0/3081131590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 msgr2=0x55d6da0746a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da0746a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 -- 172.21.15.40:0/3081131590 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef20009090 con 0x55d6da06c2d0 2023-07-14T11:20:58.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef2d99b700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fef240a30d0 tx=0x7fef240cbd90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=190cdb5bd1f7d7f1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef117fa700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:20:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef2409b240 con 0x55d6da06c2d0 2023-07-14T11:20:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fef240a10e0 con 0x55d6da06c2d0 2023-07-14T11:20:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.215+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55d6d9f184f0 con 0x55d6da06c2d0 2023-07-14T11:20:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=145) v2 ==== 24+0+0 (secure 0 0 0) 0x7fef2409f270 con 0x55d6da06c2d0 2023-07-14T11:20:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 10 client.26407 fetch_fsmap learned FSMap version 145 2023-07-14T11:20:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=145,monmap=4+}) v3 -- 0x55d6d9dd5c60 con 0x55d6da06c2d0 2023-07-14T11:20:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55d6d9f184f0 con 0x55d6da06c2d0 2023-07-14T11:20:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 145) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fef2409a730 con 0x55d6da06c2d0 2023-07-14T11:20:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 10 client.26407 fetch_fsmap finished waiting for FSMap version 145 2023-07-14T11:20:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 10 client.26407 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:20:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 20 client.26407 populate_metadata read hostname 'smithi040' 2023-07-14T11:20:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da086df0 0x55d6da089200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 4 client.26407 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:20:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.216+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55d6d9fb7eb0 con 0x55d6da086df0 2023-07-14T11:20:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.217+0000 7fef2d19a700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da086df0 0x55d6da089200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.217+0000 7fef2d19a700 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da086df0 0x55d6da089200 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=6bf744083d4b794d server_cookie=8ead0e63cc65b2cd in_seq=0 out_seq=0 2023-07-14T11:20:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.217+0000 7fef117fa700 10 client.26407 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef117fa700 1 -- 172.21.15.40:0/3081131590 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55d6d9fb7eb0 con 0x55d6da086df0 2023-07-14T11:20:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef117fa700 10 client.26407 handle_command_reply: tid=0 2023-07-14T11:20:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 client.26407 shutdown 2023-07-14T11:20:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 msgr2=0x55d6da077420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7fef240a30d0 tx=0x7fef240cbd90 comp rx=0 tx=0).stop 2023-07-14T11:20:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 shutdown_connections 2023-07-14T11:20:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da086df0 0x55d6da089200 crc :-1 s=READY pgs=105 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da074c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da077420 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 --2- 172.21.15.40:0/3081131590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da0746a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef117fa700 0 client.26407 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.218+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 >> 172.21.15.40:0/3081131590 conn(0x55d6da062da0 msgr2=0x55d6da068230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.219+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 shutdown_connections 2023-07-14T11:20:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.219+0000 7fef4354db80 1 -- 172.21.15.40:0/3081131590 wait complete. 2023-07-14T11:20:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.219+0000 7fef4354db80 20 client.26407 trim_cache size 0 max 16384 2023-07-14T11:20:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 Processor -- start 2023-07-14T11:20:58.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 -- start start 2023-07-14T11:20:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da07e4e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da076f50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6d9f184f0 con 0x55d6da06cbe0 2023-07-14T11:20:58.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6da075220 con 0x55d6da077ad0 2023-07-14T11:20:58.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef4354db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6da06dd60 con 0x55d6da06c2d0 2023-07-14T11:20:58.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52984/0 (socket says 172.21.15.40:52984) 2023-07-14T11:20:58.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef2d99b700 1 -- 172.21.15.40:0/1021314187 learned_addr learned my addr 172.21.15.40:0/1021314187 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.255+0000 7fef2e19c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da076f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da07e4e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 -- 172.21.15.40:0/1021314187 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 msgr2=0x55d6da076f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da076f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 -- 172.21.15.40:0/1021314187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 msgr2=0x55d6da07e4e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da07e4e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 -- 172.21.15.40:0/1021314187 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55d6da0670e0 con 0x55d6da06c2d0 2023-07-14T11:20:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fef24097cb0 tx=0x7fef24022c90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c7f4cab4107f8e6 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef1cbff700 1 -- 172.21.15.40:0/1021314187 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef2409b240 con 0x55d6da06c2d0 2023-07-14T11:20:58.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef1cbff700 1 -- 172.21.15.40:0/1021314187 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fef2409f270 con 0x55d6da06c2d0 2023-07-14T11:20:58.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.256+0000 7fef1cbff700 1 -- 172.21.15.40:0/1021314187 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef2409a730 con 0x55d6da06c2d0 2023-07-14T11:20:58.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 -- 172.21.15.40:0/1021314187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 msgr2=0x55d6da06f1b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7fef24097cb0 tx=0x7fef24022c90 comp rx=0 tx=0).stop 2023-07-14T11:20:58.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 -- 172.21.15.40:0/1021314187 shutdown_connections 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da076f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06c2d0 0x55d6da06f1b0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.257+0000 7fef4354db80 1 --2- 172.21.15.40:0/1021314187 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06cbe0 0x55d6da07e4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.258+0000 7fef4354db80 1 -- 172.21.15.40:0/1021314187 >> 172.21.15.40:0/1021314187 conn(0x55d6da062da0 msgr2=0x55d6da082c90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.258+0000 7fef4354db80 1 -- 172.21.15.40:0/1021314187 shutdown_connections 2023-07-14T11:20:58.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.258+0000 7fef4354db80 1 -- 172.21.15.40:0/1021314187 wait complete. 2023-07-14T11:20:58.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.258+0000 7fef4354db80 1 -- start start 2023-07-14T11:20:58.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 Processor -- start 2023-07-14T11:20:58.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 0x55d6da07e2a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da07caa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6da07cfe0 con 0x55d6da06c2d0 2023-07-14T11:20:58.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6da07d150 con 0x55d6da077ad0 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55d6da07d310 con 0x55d6da06cbe0 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.259+0000 7fef4354db80 10 client.0 _ll_register_callbacks cb 0x7fef24097580 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d19a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 0x55d6da07e2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d19a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:52988/0 (socket says 172.21.15.40:52988) 2023-07-14T11:20:58.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d19a700 1 -- 172.21.15.40:0/1090114737 learned_addr learned my addr 172.21.15.40:0/1090114737 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:20:58.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2e19c700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da07caa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 -- 172.21.15.40:0/1090114737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 msgr2=0x55d6da07c560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 -- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 msgr2=0x55d6da07caa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:20:58.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da07caa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 -- 172.21.15.40:0/1090114737 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55d6d9dd5c60 con 0x55d6da06c2d0 2023-07-14T11:20:58.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:20:58.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef2d99b700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 0x55d6da07e2a0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7fef2409af40 tx=0x7fef240997f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=bb7f750febc1d424 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:20:58.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.260+0000 7fef117fa700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:20:58.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fef240a33c0 con 0x55d6da06c2d0 2023-07-14T11:20:58.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(97..97 src has 1..97) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fef240a10d0 con 0x55d6da06c2d0 2023-07-14T11:20:58.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fef2000f040 con 0x55d6da06c2d0 2023-07-14T11:20:58.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=145) v2 ==== 24+0+0 (secure 0 0 0) 0x7fef2409b240 con 0x55d6da06c2d0 2023-07-14T11:20:58.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 10 client.16404 fetch_fsmap learned FSMap version 145 2023-07-14T11:20:58.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=145,monmap=4+}) v3 -- 0x55d6d9e13250 con 0x55d6da06c2d0 2023-07-14T11:20:58.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fef2000f040 con 0x55d6da06c2d0 2023-07-14T11:20:58.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 145) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fef2409a1a0 con 0x55d6da06c2d0 2023-07-14T11:20:58.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 10 client.16404 fetch_fsmap finished waiting for FSMap version 145 2023-07-14T11:20:58.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 10 client.16404 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:20:58.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 20 client.16404 populate_metadata read hostname 'smithi040' 2023-07-14T11:20:58.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.261+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da0dbb80 0x55d6da0ddf60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:20:58.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.262+0000 7fef4354db80 4 client.16404 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:20:58.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.262+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55d6d9f7a050 con 0x55d6da0dbb80 2023-07-14T11:20:58.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.262+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da0dbb80 0x55d6da0ddf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:20:58.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.262+0000 7fef2d19a700 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da0dbb80 0x55d6da0ddf60 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d889b85d112a507c server_cookie=be29c69958b6c51f in_seq=0 out_seq=0 2023-07-14T11:20:58.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.262+0000 7fef117fa700 10 client.16404 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:58.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef117fa700 1 -- 172.21.15.40:0/1090114737 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55d6d9f7a050 con 0x55d6da0dbb80 2023-07-14T11:20:58.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef117fa700 10 client.16404 handle_command_reply: tid=0 2023-07-14T11:20:58.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 client.16404 shutdown 2023-07-14T11:20:58.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 msgr2=0x55d6da07e2a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 0x55d6da07e2a0 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 crypto rx=0x7fef2409af40 tx=0x7fef240997f0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 shutdown_connections 2023-07-14T11:20:58.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55d6da0dbb80 0x55d6da0ddf60 crc :-1 s=READY pgs=106 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55d6da077ad0 0x55d6da07caa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55d6da06c2d0 0x55d6da07e2a0 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef117fa700 0 client.16404 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:20:58.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 --2- 172.21.15.40:0/1090114737 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55d6da06cbe0 0x55d6da07c560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 >> 172.21.15.40:0/1090114737 conn(0x55d6da062da0 msgr2=0x55d6da072ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 shutdown_connections 2023-07-14T11:20:58.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 1 -- 172.21.15.40:0/1090114737 wait complete. 2023-07-14T11:20:58.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.263+0000 7fef4354db80 20 client.16404 trim_cache size 0 max 16384 2023-07-14T11:20:58.279 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:20:58.279 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:20:58.279 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:20:58.280 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:20:58.280 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:20:58.280 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:20:58.280 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:20:58.280 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:20:58.281 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:20:58.281 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:20:58.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 msgr2=0x7fef140a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 0x7fef140a37f0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7fef2001f000 tx=0x7fef20022000 comp rx=0 tx=0).stop 2023-07-14T11:20:58.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 msgr2=0x7fef28072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:20:58.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7fef24005910 tx=0x7fef240057d0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 shutdown_connections 2023-07-14T11:20:58.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fef140a1340 0x7fef140a37f0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef28105ae0 0x7fef281a3da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fef280ffad0 0x7fef28072810 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 --2- 172.21.15.40:0/1200016285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fef28100480 0x7fef28072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:20:58.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 >> 172.21.15.40:0/1200016285 conn(0x7fef280fb5f0 msgr2=0x7fef280fbbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:20:58.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 shutdown_connections 2023-07-14T11:20:58.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:20:58.264+0000 7fef2fbff700 1 -- 172.21.15.40:0/1200016285 wait complete. 2023-07-14T11:20:58.295 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:20:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: 1420 slow requests, 5 included below; oldest blocked for > 419.040264 secs 2023-07-14T11:20:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 418.009053 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 417.976004 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 417.924433 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 417.867540 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 417.841159 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: 1417 slow requests, 5 included below; oldest blocked for > 417.163875 secs 2023-07-14T11:20:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 416.902697 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:20:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 416.892691 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:20:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 416.882655 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:20:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 416.872555 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:20:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:58 smithi130 ceph-mon[129405]: slow request 416.862499 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:20:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: 1420 slow requests, 5 included below; oldest blocked for > 419.040264 secs 2023-07-14T11:20:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 418.009053 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 417.976004 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 417.924433 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 417.867540 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 417.841159 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: 1417 slow requests, 5 included below; oldest blocked for > 417.163875 secs 2023-07-14T11:20:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 416.902697 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:20:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 416.892691 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:20:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 416.882655 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:20:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 416.872555 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:20:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:58 smithi040 ceph-mon[97954]: slow request 416.862499 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:20:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: 1420 slow requests, 5 included below; oldest blocked for > 419.040264 secs 2023-07-14T11:20:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 418.009053 seconds old, received at 2023-07-14T11:13:59.133791+0000: client_request(mds.1:1653 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 417.976004 seconds old, received at 2023-07-14T11:13:59.166840+0000: client_request(mds.1:1685 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 417.924433 seconds old, received at 2023-07-14T11:13:59.218410+0000: client_request(mds.1:1717 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 417.867540 seconds old, received at 2023-07-14T11:13:59.275304+0000: client_request(mds.1:1749 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 417.841159 seconds old, received at 2023-07-14T11:13:59.301685+0000: client_request(mds.1:1781 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:20:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: 1417 slow requests, 5 included below; oldest blocked for > 417.163875 secs 2023-07-14T11:20:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 416.902697 seconds old, received at 2023-07-14T11:14:01.194734+0000: peer_request:mds.1:1670 currently dispatched 2023-07-14T11:20:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 416.892691 seconds old, received at 2023-07-14T11:14:01.204740+0000: peer_request:mds.1:1702 currently dispatched 2023-07-14T11:20:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 416.882655 seconds old, received at 2023-07-14T11:14:01.214776+0000: peer_request:mds.1:1734 currently dispatched 2023-07-14T11:20:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 416.872555 seconds old, received at 2023-07-14T11:14:01.224876+0000: peer_request:mds.1:1766 currently dispatched 2023-07-14T11:20:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:58 smithi045 ceph-mon[128791]: slow request 416.862499 seconds old, received at 2023-07-14T11:14:01.234932+0000: peer_request:mds.1:1798 currently dispatched 2023-07-14T11:21:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:20:59 smithi130 ceph-mon[129405]: pgmap v272: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:20:59 smithi040 ceph-mon[97954]: pgmap v272: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:20:59 smithi045 ceph-mon[128791]: pgmap v272: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:01 smithi130 ceph-mon[129405]: pgmap v273: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:01 smithi040 ceph-mon[97954]: pgmap v273: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:01 smithi045 ceph-mon[128791]: pgmap v273: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: 1406 slow requests, 5 included below; oldest blocked for > 424.065933 secs 2023-07-14T11:21:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 422.838579 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 422.810989 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 422.784424 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 422.758186 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 422.731627 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: pgmap v274: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: 1403 slow requests, 5 included below; oldest blocked for > 422.163992 secs 2023-07-14T11:21:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 421.852615 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:21:04.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 421.842695 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:21:04.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 421.836963 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:21:04.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 421.831195 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:21:04.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:03 smithi130 ceph-mon[129405]: slow request 421.825508 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:21:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: 1406 slow requests, 5 included below; oldest blocked for > 424.065933 secs 2023-07-14T11:21:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 422.838579 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 422.810989 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 422.784424 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 422.758186 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 422.731627 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: pgmap v274: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: 1403 slow requests, 5 included below; oldest blocked for > 422.163992 secs 2023-07-14T11:21:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 421.852615 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:21:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 421.842695 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:21:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 421.836963 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:21:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 421.831195 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:21:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:03 smithi040 ceph-mon[97954]: slow request 421.825508 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:21:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: 1406 slow requests, 5 included below; oldest blocked for > 424.065933 secs 2023-07-14T11:21:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 422.838579 seconds old, received at 2023-07-14T11:13:59.329934+0000: client_request(mds.1:1813 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 422.810989 seconds old, received at 2023-07-14T11:13:59.357524+0000: client_request(mds.1:1845 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 422.784424 seconds old, received at 2023-07-14T11:13:59.384089+0000: client_request(mds.1:1877 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 422.758186 seconds old, received at 2023-07-14T11:13:59.410327+0000: client_request(mds.1:1909 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 422.731627 seconds old, received at 2023-07-14T11:13:59.436886+0000: client_request(mds.1:1941 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: pgmap v274: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: 1403 slow requests, 5 included below; oldest blocked for > 422.163992 secs 2023-07-14T11:21:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 421.852615 seconds old, received at 2023-07-14T11:14:01.244933+0000: peer_request:mds.1:1830 currently dispatched 2023-07-14T11:21:04.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 421.842695 seconds old, received at 2023-07-14T11:14:01.254853+0000: peer_request:mds.1:1862 currently dispatched 2023-07-14T11:21:04.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 421.836963 seconds old, received at 2023-07-14T11:14:01.260585+0000: peer_request:mds.1:1894 currently dispatched 2023-07-14T11:21:04.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 421.831195 seconds old, received at 2023-07-14T11:14:01.266353+0000: peer_request:mds.1:1926 currently dispatched 2023-07-14T11:21:04.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:03 smithi045 ceph-mon[128791]: slow request 421.825508 seconds old, received at 2023-07-14T11:14:01.272039+0000: peer_request:mds.1:1958 currently dispatched 2023-07-14T11:21:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:04 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:04 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:04 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:04 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:04 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:04 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:05 smithi130 ceph-mon[129405]: pgmap v275: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:21:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:05 smithi040 ceph-mon[97954]: pgmap v275: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:21:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:05 smithi045 ceph-mon[128791]: pgmap v275: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-07-14T11:21:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:07 smithi130 ceph-mon[129405]: pgmap v276: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:07 smithi040 ceph-mon[97954]: pgmap v276: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:07 smithi045 ceph-mon[128791]: pgmap v276: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: 1391 slow requests, 5 included below; oldest blocked for > 429.234168 secs 2023-07-14T11:21:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 427.873734 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 427.847562 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 427.821914 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 427.793570 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 427.766520 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: 1389 slow requests, 5 included below; oldest blocked for > 427.164093 secs 2023-07-14T11:21:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 426.819776 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:21:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 426.813919 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:21:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 426.808309 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:21:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 426.802599 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:21:09.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:08 smithi130 ceph-mon[129405]: slow request 426.796856 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:21:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: 1391 slow requests, 5 included below; oldest blocked for > 429.234168 secs 2023-07-14T11:21:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 427.873734 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 427.847562 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 427.821914 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 427.793570 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 427.766520 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: 1389 slow requests, 5 included below; oldest blocked for > 427.164093 secs 2023-07-14T11:21:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 426.819776 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:21:09.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 426.813919 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:21:09.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 426.808309 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:21:09.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 426.802599 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:21:09.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:08 smithi040 ceph-mon[97954]: slow request 426.796856 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:21:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: 1391 slow requests, 5 included below; oldest blocked for > 429.234168 secs 2023-07-14T11:21:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 427.873734 seconds old, received at 2023-07-14T11:13:59.463014+0000: client_request(mds.1:1973 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 427.847562 seconds old, received at 2023-07-14T11:13:59.489186+0000: client_request(mds.1:2005 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 427.821914 seconds old, received at 2023-07-14T11:13:59.514834+0000: client_request(mds.1:2037 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 427.793570 seconds old, received at 2023-07-14T11:13:59.543178+0000: client_request(mds.1:2069 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 427.766520 seconds old, received at 2023-07-14T11:13:59.570228+0000: client_request(mds.1:2101 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: 1389 slow requests, 5 included below; oldest blocked for > 427.164093 secs 2023-07-14T11:21:09.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 426.819776 seconds old, received at 2023-07-14T11:14:01.277873+0000: peer_request:mds.1:1990 currently dispatched 2023-07-14T11:21:09.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 426.813919 seconds old, received at 2023-07-14T11:14:01.283730+0000: peer_request:mds.1:2022 currently dispatched 2023-07-14T11:21:09.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 426.808309 seconds old, received at 2023-07-14T11:14:01.289340+0000: peer_request:mds.1:2054 currently dispatched 2023-07-14T11:21:09.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 426.802599 seconds old, received at 2023-07-14T11:14:01.295050+0000: peer_request:mds.1:2086 currently dispatched 2023-07-14T11:21:09.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:08 smithi045 ceph-mon[128791]: slow request 426.796856 seconds old, received at 2023-07-14T11:14:01.300793+0000: peer_request:mds.1:2118 currently dispatched 2023-07-14T11:21:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:09 smithi130 ceph-mon[129405]: pgmap v277: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:09 smithi130 ceph-mon[129405]: osdmap e98: 12 total, 12 up, 12 in 2023-07-14T11:21:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:09 smithi040 ceph-mon[97954]: pgmap v277: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:09 smithi040 ceph-mon[97954]: osdmap e98: 12 total, 12 up, 12 in 2023-07-14T11:21:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:09 smithi045 ceph-mon[128791]: pgmap v277: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 45 KiB/s wr, 22 op/s 2023-07-14T11:21:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:09 smithi045 ceph-mon[128791]: osdmap e98: 12 total, 12 up, 12 in 2023-07-14T11:21:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:11 smithi130 ceph-mon[129405]: pgmap v279: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:11 smithi040 ceph-mon[97954]: pgmap v279: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:11 smithi045 ceph-mon[128791]: pgmap v279: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: 1376 slow requests, 5 included below; oldest blocked for > 434.466226 secs 2023-07-14T11:21:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 432.971817 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 432.939733 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 432.914004 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 432.888745 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 432.863692 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: pgmap v280: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: 1374 slow requests, 5 included below; oldest blocked for > 432.164154 secs 2023-07-14T11:21:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 431.791260 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:21:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 431.785518 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:21:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 431.779847 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:21:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 431.774082 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:21:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:13 smithi130 ceph-mon[129405]: slow request 431.768062 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:21:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: 1376 slow requests, 5 included below; oldest blocked for > 434.466226 secs 2023-07-14T11:21:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 432.971817 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 432.939733 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 432.914004 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 432.888745 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 432.863692 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: pgmap v280: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: 1374 slow requests, 5 included below; oldest blocked for > 432.164154 secs 2023-07-14T11:21:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 431.791260 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:21:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 431.785518 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:21:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 431.779847 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:21:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 431.774082 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:21:14.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:13 smithi040 ceph-mon[97954]: slow request 431.768062 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:21:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: 1376 slow requests, 5 included below; oldest blocked for > 434.466226 secs 2023-07-14T11:21:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 432.971817 seconds old, received at 2023-07-14T11:13:59.596989+0000: client_request(mds.1:2133 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 432.939733 seconds old, received at 2023-07-14T11:13:59.629073+0000: client_request(mds.1:2165 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 432.914004 seconds old, received at 2023-07-14T11:13:59.654801+0000: client_request(mds.1:2197 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 432.888745 seconds old, received at 2023-07-14T11:13:59.680061+0000: client_request(mds.1:2229 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 432.863692 seconds old, received at 2023-07-14T11:13:59.705114+0000: client_request(mds.1:2261 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: pgmap v280: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 54 KiB/s wr, 27 op/s 2023-07-14T11:21:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: 1374 slow requests, 5 included below; oldest blocked for > 432.164154 secs 2023-07-14T11:21:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 431.791260 seconds old, received at 2023-07-14T11:14:01.306449+0000: peer_request:mds.1:2150 currently dispatched 2023-07-14T11:21:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 431.785518 seconds old, received at 2023-07-14T11:14:01.312192+0000: peer_request:mds.1:2182 currently dispatched 2023-07-14T11:21:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 431.779847 seconds old, received at 2023-07-14T11:14:01.317862+0000: peer_request:mds.1:2214 currently dispatched 2023-07-14T11:21:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 431.774082 seconds old, received at 2023-07-14T11:14:01.323628+0000: peer_request:mds.1:2246 currently dispatched 2023-07-14T11:21:14.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:13 smithi045 ceph-mon[128791]: slow request 431.768062 seconds old, received at 2023-07-14T11:14:01.329647+0000: peer_request:mds.1:2278 currently dispatched 2023-07-14T11:21:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:15 smithi130 ceph-mon[129405]: osdmap e99: 12 total, 12 up, 12 in 2023-07-14T11:21:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:15 smithi130 ceph-mon[129405]: pgmap v282: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:15.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:15.794 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:15 smithi040 ceph-mon[97954]: osdmap e99: 12 total, 12 up, 12 in 2023-07-14T11:21:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:15 smithi040 ceph-mon[97954]: pgmap v282: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:15.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:15 smithi045 ceph-mon[128791]: osdmap e99: 12 total, 12 up, 12 in 2023-07-14T11:21:15.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:15 smithi045 ceph-mon[128791]: pgmap v282: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:15.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:17.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:17.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:17 smithi045 ceph-mon[128791]: pgmap v283: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:17 smithi130 ceph-mon[129405]: pgmap v283: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:17 smithi040 ceph-mon[97954]: pgmap v283: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:21:18.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: 1361 slow requests, 5 included below; oldest blocked for > 439.546204 secs 2023-07-14T11:21:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 437.915238 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 437.876191 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 437.845685 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:18.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 437.819497 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 437.793055 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: 1359 slow requests, 5 included below; oldest blocked for > 437.164245 secs 2023-07-14T11:21:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 436.762127 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:21:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 436.756238 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:21:18.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 436.750284 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:21:18.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 436.744329 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:21:18.817 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:18 smithi045 ceph-mon[128791]: slow request 436.738452 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:21:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: 1361 slow requests, 5 included below; oldest blocked for > 439.546204 secs 2023-07-14T11:21:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 437.915238 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 437.876191 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 437.845685 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 437.819497 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 437.793055 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: 1359 slow requests, 5 included below; oldest blocked for > 437.164245 secs 2023-07-14T11:21:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 436.762127 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:21:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 436.756238 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:21:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 436.750284 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:21:19.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 436.744329 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:21:19.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:18 smithi130 ceph-mon[129405]: slow request 436.738452 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:21:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: 1361 slow requests, 5 included below; oldest blocked for > 439.546204 secs 2023-07-14T11:21:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 437.915238 seconds old, received at 2023-07-14T11:13:59.733546+0000: client_request(mds.1:2293 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 437.876191 seconds old, received at 2023-07-14T11:13:59.772593+0000: client_request(mds.1:2325 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 437.845685 seconds old, received at 2023-07-14T11:13:59.803099+0000: client_request(mds.1:2357 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 437.819497 seconds old, received at 2023-07-14T11:13:59.829287+0000: client_request(mds.1:2389 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 437.793055 seconds old, received at 2023-07-14T11:13:59.855729+0000: client_request(mds.1:2421 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: 1359 slow requests, 5 included below; oldest blocked for > 437.164245 secs 2023-07-14T11:21:19.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 436.762127 seconds old, received at 2023-07-14T11:14:01.335675+0000: peer_request:mds.1:2310 currently dispatched 2023-07-14T11:21:19.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 436.756238 seconds old, received at 2023-07-14T11:14:01.341564+0000: peer_request:mds.1:2342 currently dispatched 2023-07-14T11:21:19.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 436.750284 seconds old, received at 2023-07-14T11:14:01.347518+0000: peer_request:mds.1:2374 currently dispatched 2023-07-14T11:21:19.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 436.744329 seconds old, received at 2023-07-14T11:14:01.353472+0000: peer_request:mds.1:2406 currently dispatched 2023-07-14T11:21:19.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:18 smithi040 ceph-mon[97954]: slow request 436.738452 seconds old, received at 2023-07-14T11:14:01.359349+0000: peer_request:mds.1:2438 currently dispatched 2023-07-14T11:21:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:19 smithi130 ceph-mon[129405]: pgmap v284: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 206 B/s wr, 10 op/s 2023-07-14T11:21:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:19 smithi040 ceph-mon[97954]: pgmap v284: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 206 B/s wr, 10 op/s 2023-07-14T11:21:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:19 smithi045 ceph-mon[128791]: pgmap v284: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 206 B/s wr, 10 op/s 2023-07-14T11:21:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:21 smithi130 ceph-mon[129405]: pgmap v285: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:21 smithi040 ceph-mon[97954]: pgmap v285: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:21 smithi045 ceph-mon[128791]: pgmap v285: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: pgmap v286: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: 1346 slow requests, 5 included below; oldest blocked for > 444.619445 secs 2023-07-14T11:21:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 442.840303 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 442.814160 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 442.787443 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 442.759007 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 442.731425 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: 1345 slow requests, 5 included below; oldest blocked for > 442.164345 secs 2023-07-14T11:21:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 441.732658 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:21:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 441.726814 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:21:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 441.720945 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:21:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 441.715012 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:21:24.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:23 smithi130 ceph-mon[129405]: slow request 441.709128 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:21:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: pgmap v286: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: 1346 slow requests, 5 included below; oldest blocked for > 444.619445 secs 2023-07-14T11:21:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 442.840303 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 442.814160 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 442.787443 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 442.759007 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 442.731425 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: 1345 slow requests, 5 included below; oldest blocked for > 442.164345 secs 2023-07-14T11:21:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 441.732658 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:21:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 441.726814 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:21:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 441.720945 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:21:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 441.715012 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:21:24.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:23 smithi040 ceph-mon[97954]: slow request 441.709128 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:21:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: pgmap v286: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:21:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: 1346 slow requests, 5 included below; oldest blocked for > 444.619445 secs 2023-07-14T11:21:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 442.840303 seconds old, received at 2023-07-14T11:13:59.881721+0000: client_request(mds.1:2453 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 442.814160 seconds old, received at 2023-07-14T11:13:59.907865+0000: client_request(mds.1:2485 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 442.787443 seconds old, received at 2023-07-14T11:13:59.934582+0000: client_request(mds.1:2517 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 442.759007 seconds old, received at 2023-07-14T11:13:59.963018+0000: client_request(mds.1:2549 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 442.731425 seconds old, received at 2023-07-14T11:13:59.990599+0000: client_request(mds.1:2581 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: 1345 slow requests, 5 included below; oldest blocked for > 442.164345 secs 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 441.732658 seconds old, received at 2023-07-14T11:14:01.365243+0000: peer_request:mds.1:2470 currently dispatched 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 441.726814 seconds old, received at 2023-07-14T11:14:01.371086+0000: peer_request:mds.1:2502 currently dispatched 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 441.720945 seconds old, received at 2023-07-14T11:14:01.376956+0000: peer_request:mds.1:2534 currently dispatched 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 441.715012 seconds old, received at 2023-07-14T11:14:01.382889+0000: peer_request:mds.1:2566 currently dispatched 2023-07-14T11:21:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:23 smithi045 ceph-mon[128791]: slow request 441.709128 seconds old, received at 2023-07-14T11:14:01.388773+0000: peer_request:mds.1:2598 currently dispatched 2023-07-14T11:21:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:24 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:24 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:24 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:24 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:24 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:24 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:25 smithi130 ceph-mon[129405]: pgmap v287: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:21:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:25 smithi040 ceph-mon[97954]: pgmap v287: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:21:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:25 smithi045 ceph-mon[128791]: pgmap v287: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 101 B/s wr, 15 op/s 2023-07-14T11:21:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:27 smithi130 ceph-mon[129405]: pgmap v288: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:27 smithi040 ceph-mon[97954]: pgmap v288: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:27 smithi045 ceph-mon[128791]: pgmap v288: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:28.296 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:21:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: 1333 slow requests, 5 included below; oldest blocked for > 449.638685 secs 2023-07-14T11:21:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 447.723139 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 447.697397 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 447.670805 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 447.644049 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 447.617919 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: 1331 slow requests, 5 included below; oldest blocked for > 447.164425 secs 2023-07-14T11:21:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 446.703183 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:21:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 446.697277 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:21:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 446.691517 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:21:29.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 446.685569 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:21:29.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:28 smithi130 ceph-mon[129405]: slow request 446.679683 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:21:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: 1333 slow requests, 5 included below; oldest blocked for > 449.638685 secs 2023-07-14T11:21:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 447.723139 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 447.697397 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 447.670805 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 447.644049 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 447.617919 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: 1331 slow requests, 5 included below; oldest blocked for > 447.164425 secs 2023-07-14T11:21:29.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 446.703183 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:21:29.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 446.697277 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:21:29.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 446.691517 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:21:29.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 446.685569 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:21:29.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:28 smithi040 ceph-mon[97954]: slow request 446.679683 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:21:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: 1333 slow requests, 5 included below; oldest blocked for > 449.638685 secs 2023-07-14T11:21:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 447.723139 seconds old, received at 2023-07-14T11:14:00.018126+0000: client_request(mds.1:2613 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 447.697397 seconds old, received at 2023-07-14T11:14:00.043868+0000: client_request(mds.1:2645 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 447.670805 seconds old, received at 2023-07-14T11:14:00.070460+0000: client_request(mds.1:2677 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 447.644049 seconds old, received at 2023-07-14T11:14:00.097216+0000: client_request(mds.1:2709 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 447.617919 seconds old, received at 2023-07-14T11:14:00.123346+0000: client_request(mds.1:2741 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: 1331 slow requests, 5 included below; oldest blocked for > 447.164425 secs 2023-07-14T11:21:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 446.703183 seconds old, received at 2023-07-14T11:14:01.394798+0000: peer_request:mds.1:2630 currently dispatched 2023-07-14T11:21:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 446.697277 seconds old, received at 2023-07-14T11:14:01.400703+0000: peer_request:mds.1:2662 currently dispatched 2023-07-14T11:21:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 446.691517 seconds old, received at 2023-07-14T11:14:01.406463+0000: peer_request:mds.1:2694 currently dispatched 2023-07-14T11:21:29.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 446.685569 seconds old, received at 2023-07-14T11:14:01.412412+0000: peer_request:mds.1:2726 currently dispatched 2023-07-14T11:21:29.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:28 smithi045 ceph-mon[128791]: slow request 446.679683 seconds old, received at 2023-07-14T11:14:01.418298+0000: peer_request:mds.1:2758 currently dispatched 2023-07-14T11:21:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:29 smithi130 ceph-mon[129405]: pgmap v289: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:29 smithi040 ceph-mon[97954]: pgmap v289: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:29 smithi045 ceph-mon[128791]: pgmap v289: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:30.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 -- 172.21.15.40:0/2196286689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c101470 msgr2=0x7f788c109b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 --2- 172.21.15.40:0/2196286689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c101470 0x7f788c109b40 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f78800098f0 tx=0x7f7880009c00 comp rx=0 tx=0).stop 2023-07-14T11:21:30.750 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 -- 172.21.15.40:0/2196286689 shutdown_connections 2023-07-14T11:21:30.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 --2- 172.21.15.40:0/2196286689 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c10a080 0x7f788c10c4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 --2- 172.21.15.40:0/2196286689 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c100b60 0x7f788c100f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 --2- 172.21.15.40:0/2196286689 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c101470 0x7f788c109b40 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 -- 172.21.15.40:0/2196286689 >> 172.21.15.40:0/2196286689 conn(0x7f788c0fa4b0 msgr2=0x7f788c0fc8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.751 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 -- 172.21.15.40:0/2196286689 shutdown_connections 2023-07-14T11:21:30.752 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.747+0000 7f7891bff700 1 -- 172.21.15.40:0/2196286689 wait complete. 2023-07-14T11:21:30.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.749+0000 7f7891bff700 1 Processor -- start 2023-07-14T11:21:30.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.749+0000 7f7891bff700 1 -- start start 2023-07-14T11:21:30.753 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 0x7f788c0feed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c10a080 0x7f788c1a3e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f788c0ff700 con 0x7f788c100b60 2023-07-14T11:21:30.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f788c1a4370 con 0x7f788c101470 2023-07-14T11:21:30.754 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f7891bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f788c1a44b0 con 0x7f788c10a080 2023-07-14T11:21:30.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f788affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 0x7f788c0feed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:42986/0 (socket says 172.21.15.40:42986) 2023-07-14T11:21:30.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f788b7fe700 1 -- 172.21.15.40:0/1713940181 learned_addr learned my addr 172.21.15.40:0/1713940181 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:21:30.755 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.750+0000 7f788bfff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c10a080 0x7f788c1a3e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 -- 172.21.15.40:0/1713940181 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c10a080 msgr2=0x7f788c1a3e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c10a080 0x7f788c1a3e30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 -- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 msgr2=0x7f788c0fe990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.756 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 -- 172.21.15.40:0/1713940181 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7880009520 con 0x7f788c101470 2023-07-14T11:21:30.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:21:30.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.751+0000 7f788affd700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 0x7f788c0feed0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f7880009810 tx=0x7f788000fa00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.757 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.753+0000 7f7888ff9700 1 -- 172.21.15.40:0/1713940181 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f788001c070 con 0x7f788c101470 2023-07-14T11:21:30.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.753+0000 7f7888ff9700 1 -- 172.21.15.40:0/1713940181 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f788000bba0 con 0x7f788c101470 2023-07-14T11:21:30.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.753+0000 7f7888ff9700 1 -- 172.21.15.40:0/1713940181 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7880003970 con 0x7f788c101470 2023-07-14T11:21:30.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.753+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f788c1a46b0 con 0x7f788c101470 2023-07-14T11:21:30.758 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.753+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f788c1a4ae0 con 0x7f788c101470 2023-07-14T11:21:30.760 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.758+0000 7f78a54b8b80 1 Processor -- start 2023-07-14T11:21:30.761 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 -- start start 2023-07-14T11:21:30.762 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f7888ff9700 1 -- 172.21.15.40:0/1713940181 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f7880020030 con 0x7f788c101470 2023-07-14T11:21:30.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.763 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b9be0 0x5594a46ccda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46cd2e0 0x5594a46cf6c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a45bfcb0 con 0x5594a46cd2e0 2023-07-14T11:21:30.764 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a4612a40 con 0x5594a46b92d0 2023-07-14T11:21:30.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a460bed0 con 0x5594a46b9be0 2023-07-14T11:21:30.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.759+0000 7f788bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50506/0 (socket says 172.21.15.40:50506) 2023-07-14T11:21:30.765 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 -- 172.21.15.40:0/1692089215 learned_addr learned my addr 172.21.15.40:0/1692089215 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:21:30.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b9be0 0x5594a46ccda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788affd700 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46cd2e0 0x5594a46cf6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.766 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 -- 172.21.15.40:0/1692089215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b9be0 msgr2=0x5594a46ccda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b9be0 0x5594a46ccda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 -- 172.21.15.40:0/1692089215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46cd2e0 msgr2=0x5594a46cf6c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46cd2e0 0x5594a46cf6c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.767 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.760+0000 7f788bfff700 1 -- 172.21.15.40:0/1692089215 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5594a4439de0 con 0x5594a46b92d0 2023-07-14T11:21:30.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f788bfff700 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f7884011600 tx=0x7f7884009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=8178f079d005fff8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f7871ffb700 1 -- 172.21.15.40:0/1692089215 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f788400e040 con 0x5594a46b92d0 2023-07-14T11:21:30.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f7871ffb700 1 -- 172.21.15.40:0/1692089215 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f788400f040 con 0x5594a46b92d0 2023-07-14T11:21:30.768 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f7871ffb700 1 -- 172.21.15.40:0/1692089215 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7884015870 con 0x5594a46b92d0 2023-07-14T11:21:30.769 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f7888ff9700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 0x7f78740a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.761+0000 7f7888ff9700 1 -- 172.21.15.40:0/1713940181 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f78800cde10 con 0x7f788c101470 2023-07-14T11:21:30.771 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 0x7f78740a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1692089215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 msgr2=0x5594a46b96a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f7884011600 tx=0x7f7884009920 comp rx=0 tx=0).stop 2023-07-14T11:21:30.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1692089215 shutdown_connections 2023-07-14T11:21:30.772 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46cd2e0 0x5594a46cf6c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46b96a0 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1692089215 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b9be0 0x5594a46ccda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1692089215 >> 172.21.15.40:0/1692089215 conn(0x5594a46afda0 msgr2=0x5594a46b56a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.762+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1692089215 shutdown_connections 2023-07-14T11:21:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.763+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1692089215 wait complete. 2023-07-14T11:21:30.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.763+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 0x7f78740a37f0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f787c005950 tx=0x7f787c009310 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.763+0000 7f78a54b8b80 1 -- start start 2023-07-14T11:21:30.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.764+0000 7f78a54b8b80 1 Processor -- start 2023-07-14T11:21:30.774 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.764+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46ca010 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46ca750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a45abac0 con 0x5594a46b9be0 2023-07-14T11:21:30.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a45ab7c0 con 0x5594a46cd2e0 2023-07-14T11:21:30.775 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a448f920 con 0x5594a46b92d0 2023-07-14T11:21:30.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f78a54b8b80 10 client.0 _ll_register_callbacks cb 0x5594a45a9760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:21:30.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:48788/0 (socket says 172.21.15.40:48788) 2023-07-14T11:21:30.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 -- 172.21.15.40:0/4089541900 learned_addr learned my addr 172.21.15.40:0/4089541900 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:21:30.776 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 -- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 msgr2=0x5594a46ca750 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:21:30.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46ca750 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 -- 172.21.15.40:0/4089541900 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 msgr2=0x5594a46ca010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46ca010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.777 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.765+0000 7f788b7fe700 1 -- 172.21.15.40:0/4089541900 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78840112b0 con 0x5594a46b92d0 2023-07-14T11:21:30.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.766+0000 7f788b7fe700 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f787c0046b0 tx=0x7f787c0047c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=2016697f3e771fec server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.766+0000 7f786affd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:21:30.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.766+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f787c017070 con 0x5594a46b92d0 2023-07-14T11:21:30.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.766+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5594a45654f0 con 0x5594a46b92d0 2023-07-14T11:21:30.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.766+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f787c012020 con 0x5594a46b92d0 2023-07-14T11:21:30.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=158) v2 ==== 24+0+0 (secure 0 0 0) 0x7f787c0121e0 con 0x5594a46b92d0 2023-07-14T11:21:30.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 10 client.26425 fetch_fsmap learned FSMap version 158 2023-07-14T11:21:30.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=158,monmap=4+}) v3 -- 0x5594a4422c60 con 0x5594a46b92d0 2023-07-14T11:21:30.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5594a45654f0 con 0x5594a46b92d0 2023-07-14T11:21:30.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 158) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f787c011670 con 0x5594a46b92d0 2023-07-14T11:21:30.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 10 client.26425 fetch_fsmap finished waiting for FSMap version 158 2023-07-14T11:21:30.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 10 client.26425 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:21:30.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 20 client.26425 populate_metadata read hostname 'smithi040' 2023-07-14T11:21:30.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a46bee30 0x5594a46c1240 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.781 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 4 client.26425 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:21:30.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.767+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x5594a4604eb0 con 0x5594a46bee30 2023-07-14T11:21:30.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.768+0000 7f788affd700 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a46bee30 0x5594a46c1240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.768+0000 7f786affd700 10 client.26425 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:21:30.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.768+0000 7f788affd700 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a46bee30 0x5594a46c1240 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3c65a83a7a7951a9 server_cookie=ada700992797368a in_seq=0 out_seq=0 2023-07-14T11:21:30.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.769+0000 7f786affd700 1 -- 172.21.15.40:0/4089541900 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x5594a4604eb0 con 0x5594a46bee30 2023-07-14T11:21:30.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.769+0000 7f786affd700 10 client.26425 handle_command_reply: tid=0 2023-07-14T11:21:30.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.769+0000 7f78a54b8b80 1 client.26425 shutdown 2023-07-14T11:21:30.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.769+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 msgr2=0x5594a46ccd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.769+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f787c0046b0 tx=0x7f787c0047c0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 shutdown_connections 2023-07-14T11:21:30.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a46bee30 0x5594a46c1240 crc :-1 s=READY pgs=107 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46ca750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46ccd90 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/4089541900 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46ca010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 >> 172.21.15.40:0/4089541900 conn(0x5594a46afda0 msgr2=0x5594a46b5290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f786affd700 0 client.26425 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:21:30.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 shutdown_connections 2023-07-14T11:21:30.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 1 -- 172.21.15.40:0/4089541900 wait complete. 2023-07-14T11:21:30.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.770+0000 7f78a54b8b80 20 client.26425 trim_cache size 0 max 16384 2023-07-14T11:21:30.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.788+0000 7f78a54b8b80 1 Processor -- start 2023-07-14T11:21:30.790 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.788+0000 7f78a54b8b80 1 -- start start 2023-07-14T11:21:30.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.788+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46de2d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.788+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46d83e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.791 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a46c1560 con 0x5594a46b9be0 2023-07-14T11:21:30.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a46c9580 con 0x5594a46cd2e0 2023-07-14T11:21:30.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a46c9740 con 0x5594a46b92d0 2023-07-14T11:21:30.792 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50522/0 (socket says 172.21.15.40:50522) 2023-07-14T11:21:30.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788bfff700 1 -- 172.21.15.40:0/3533702651 learned_addr learned my addr 172.21.15.40:0/3533702651 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:21:30.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46de2d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.793 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788affd700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46d83e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.795 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 -- 172.21.15.40:0/3533702651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 msgr2=0x5594a46cc8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 -- 172.21.15.40:0/3533702651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 msgr2=0x5594a46d83e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46d83e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.796 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788b7fe700 1 -- 172.21.15.40:0/3533702651 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5594a4460250 con 0x5594a46b92d0 2023-07-14T11:21:30.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.789+0000 7f788bfff700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:21:30.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.790+0000 7f788b7fe700 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46de2d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f787c0095c0 tx=0x7f787c005a60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=4325a051a9eb5788 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.797 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.790+0000 7f78727fc700 1 -- 172.21.15.40:0/3533702651 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f787c017040 con 0x5594a46b92d0 2023-07-14T11:21:30.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.790+0000 7f78727fc700 1 -- 172.21.15.40:0/3533702651 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f787c003e20 con 0x5594a46b92d0 2023-07-14T11:21:30.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.790+0000 7f78727fc700 1 -- 172.21.15.40:0/3533702651 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f787c01e680 con 0x5594a46b92d0 2023-07-14T11:21:30.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- 172.21.15.40:0/3533702651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 msgr2=0x5594a46de2d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46de2d0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f787c0095c0 tx=0x7f787c005a60 comp rx=0 tx=0).stop 2023-07-14T11:21:30.798 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- 172.21.15.40:0/3533702651 shutdown_connections 2023-07-14T11:21:30.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46cd2e0 0x5594a46cc8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46b92d0 0x5594a46de2d0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.799 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/3533702651 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46d83e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- 172.21.15.40:0/3533702651 >> 172.21.15.40:0/3533702651 conn(0x5594a46afda0 msgr2=0x5594a46d7cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- 172.21.15.40:0/3533702651 shutdown_connections 2023-07-14T11:21:30.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- 172.21.15.40:0/3533702651 wait complete. 2023-07-14T11:21:30.800 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 -- start start 2023-07-14T11:21:30.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.791+0000 7f78a54b8b80 1 Processor -- start 2023-07-14T11:21:30.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46dc7d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 0x5594a46dcd10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.801 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a4728500 con 0x5594a46b9be0 2023-07-14T11:21:30.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a4728640 con 0x5594a46b92d0 2023-07-14T11:21:30.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5594a4728780 con 0x5594a46cd2e0 2023-07-14T11:21:30.802 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f78a54b8b80 10 client.0 _ll_register_callbacks cb 0x7f787c004470 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:21:30.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f788b7fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:50534/0 (socket says 172.21.15.40:50534) 2023-07-14T11:21:30.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.792+0000 7f788b7fe700 1 -- 172.21.15.40:0/1276131074 learned_addr learned my addr 172.21.15.40:0/1276131074 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:21:30.803 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788bfff700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 0x5594a46dcd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788affd700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46dc7d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 -- 172.21.15.40:0/1276131074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 msgr2=0x5594a46dcd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 0x5594a46dcd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 -- 172.21.15.40:0/1276131074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 msgr2=0x5594a46dc7d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:21:30.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46dc7d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 -- 172.21.15.40:0/1276131074 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x5594a4439de0 con 0x5594a46b92d0 2023-07-14T11:21:30.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788bfff700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 0x5594a46dcd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:21:30.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788affd700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46dc7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:21:30.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f788b7fe700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f787c011a20 tx=0x7f787c011ea0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=32952fd5769acb66 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:21:30.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.793+0000 7f786affd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:21:30.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f787c017040 con 0x5594a46b92d0 2023-07-14T11:21:30.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f787c0122a0 con 0x5594a46b92d0 2023-07-14T11:21:30.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f788400f040 con 0x5594a46b92d0 2023-07-14T11:21:30.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=158) v2 ==== 24+0+0 (secure 0 0 0) 0x7f787c005250 con 0x5594a46b92d0 2023-07-14T11:21:30.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f78a54b8b80 10 client.26465 fetch_fsmap learned FSMap version 158 2023-07-14T11:21:30.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.794+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=158,monmap=4+}) v3 -- 0x5594a4422c60 con 0x5594a46b92d0 2023-07-14T11:21:30.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f788400f040 con 0x5594a46b92d0 2023-07-14T11:21:30.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 158) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f787c01e6a0 con 0x5594a46b92d0 2023-07-14T11:21:30.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 10 client.26465 fetch_fsmap finished waiting for FSMap version 158 2023-07-14T11:21:30.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 10 client.26465 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:21:30.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 20 client.26465 populate_metadata read hostname 'smithi040' 2023-07-14T11:21:30.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a4729af0 0x5594a472bed0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:21:30.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 4 client.26465 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:21:30.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5594a45c7050 con 0x5594a4729af0 2023-07-14T11:21:30.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.795+0000 7f788affd700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a4729af0 0x5594a472bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:21:30.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f788affd700 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a4729af0 0x5594a472bed0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=2300e22611fb7830 server_cookie=becfb5ebf2252b22 in_seq=0 out_seq=0 2023-07-14T11:21:30.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f786affd700 10 client.26465 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:21:30.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f786affd700 1 -- 172.21.15.40:0/1276131074 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x5594a45c7050 con 0x5594a4729af0 2023-07-14T11:21:30.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f786affd700 10 client.26465 handle_command_reply: tid=0 2023-07-14T11:21:30.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 client.26465 shutdown 2023-07-14T11:21:30.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 msgr2=0x5594a46dc290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f787c011a20 tx=0x7f787c011ea0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 shutdown_connections 2023-07-14T11:21:30.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5594a4729af0 0x5594a472bed0 crc :-1 s=READY pgs=108 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5594a46cd2e0 0x5594a46dcd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5594a46b92d0 0x5594a46dc290 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 --2- 172.21.15.40:0/1276131074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5594a46b9be0 0x5594a46dc7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 >> 172.21.15.40:0/1276131074 conn(0x5594a46afda0 msgr2=0x5594a46c8e00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.796+0000 7f786affd700 0 client.26465 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:21:30.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.797+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 shutdown_connections 2023-07-14T11:21:30.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.797+0000 7f78a54b8b80 1 -- 172.21.15.40:0/1276131074 wait complete. 2023-07-14T11:21:30.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.797+0000 7f78a54b8b80 20 client.26465 trim_cache size 0 max 16384 2023-07-14T11:21:30.817 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:21:30.817 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:21:30.818 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:21:30.818 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:21:30.818 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:21:30.819 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:21:30.819 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:21:30.819 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:21:30.819 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:21:30.819 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:21:30.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 msgr2=0x7f78740a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 0x7f78740a37f0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f787c005950 tx=0x7f787c009310 comp rx=0 tx=0).stop 2023-07-14T11:21:30.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 msgr2=0x7f788c0feed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:21:30.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 0x7f788c0feed0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f7880009810 tx=0x7f788000fa00 comp rx=0 tx=0).stop 2023-07-14T11:21:30.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 shutdown_connections 2023-07-14T11:21:30.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f78740a1340 0x7f78740a37f0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f788c10a080 0x7f788c1a3e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f788c100b60 0x7f788c0fe990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 --2- 172.21.15.40:0/1713940181 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f788c101470 0x7f788c0feed0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:21:30.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 >> 172.21.15.40:0/1713940181 conn(0x7f788c0fa4b0 msgr2=0x7f788c10c250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:21:30.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 shutdown_connections 2023-07-14T11:21:30.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:21:30.798+0000 7f7891bff700 1 -- 172.21.15.40:0/1713940181 wait complete. 2023-07-14T11:21:31.024 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:21:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:31 smithi130 ceph-mon[129405]: pgmap v290: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:31 smithi040 ceph-mon[97954]: pgmap v290: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:31 smithi045 ceph-mon[128791]: pgmap v290: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:32 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:33.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:32 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:32 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:33.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:32 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:32 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:32 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: pgmap v291: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: 1319 slow requests, 5 included below; oldest blocked for > 454.702506 secs 2023-07-14T11:21:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 452.655191 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 452.629152 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 452.602923 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 452.575102 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 453.732783 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: 1318 slow requests, 5 included below; oldest blocked for > 452.164530 secs 2023-07-14T11:21:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 451.673918 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:21:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 451.668001 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:21:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 451.662188 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:21:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 451.930867 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:21:34.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:33 smithi130 ceph-mon[129405]: slow request 451.922788 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:21:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: pgmap v291: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: 1319 slow requests, 5 included below; oldest blocked for > 454.702506 secs 2023-07-14T11:21:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 452.655191 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 452.629152 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 452.602923 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 452.575102 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 453.732783 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: 1318 slow requests, 5 included below; oldest blocked for > 452.164530 secs 2023-07-14T11:21:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 451.673918 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:21:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 451.668001 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:21:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 451.662188 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:21:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 451.930867 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:21:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:33 smithi040 ceph-mon[97954]: slow request 451.922788 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:21:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: pgmap v291: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: 1319 slow requests, 5 included below; oldest blocked for > 454.702506 secs 2023-07-14T11:21:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 452.655191 seconds old, received at 2023-07-14T11:14:00.149895+0000: client_request(mds.1:2773 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 452.629152 seconds old, received at 2023-07-14T11:14:00.175934+0000: client_request(mds.1:2805 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 452.602923 seconds old, received at 2023-07-14T11:14:00.202163+0000: client_request(mds.1:2837 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 452.575102 seconds old, received at 2023-07-14T11:14:00.229984+0000: client_request(mds.1:2869 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 453.732783 seconds old, received at 2023-07-14T11:13:59.072303+0000: client_request(mds.1:1590 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: 1318 slow requests, 5 included below; oldest blocked for > 452.164530 secs 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 451.673918 seconds old, received at 2023-07-14T11:14:01.424169+0000: peer_request:mds.1:2790 currently dispatched 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 451.668001 seconds old, received at 2023-07-14T11:14:01.430086+0000: peer_request:mds.1:2822 currently dispatched 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 451.662188 seconds old, received at 2023-07-14T11:14:01.435899+0000: peer_request:mds.1:2854 currently dispatched 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 451.930867 seconds old, received at 2023-07-14T11:14:01.167219+0000: peer_request:mds.1:1575 currently dispatched 2023-07-14T11:21:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:33 smithi045 ceph-mon[128791]: slow request 451.922788 seconds old, received at 2023-07-14T11:14:01.175299+0000: peer_request:mds.1:1607 currently dispatched 2023-07-14T11:21:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:35 smithi130 ceph-mon[129405]: pgmap v292: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:35 smithi040 ceph-mon[97954]: pgmap v292: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:35 smithi045 ceph-mon[128791]: pgmap v292: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:37.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:36 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:21:37.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:36 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:37.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:36 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:37.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:36 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:21:37.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:36 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:37.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:36 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:37.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:36 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:21:37.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:36 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:37.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:36 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:37 smithi130 ceph-mon[129405]: pgmap v293: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:37 smithi040 ceph-mon[97954]: pgmap v293: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:37 smithi045 ceph-mon[128791]: pgmap v293: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: 1305 slow requests, 5 included below; oldest blocked for > 459.752194 secs 2023-07-14T11:21:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 458.749082 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 458.719956 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 458.686372 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 458.632278 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 458.578557 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: 1304 slow requests, 5 included below; oldest blocked for > 457.164608 secs 2023-07-14T11:21:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 456.913004 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:21:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 456.903125 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:21:39.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 456.893111 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:21:39.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 456.883082 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:21:39.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:38 smithi130 ceph-mon[129405]: slow request 456.872986 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:21:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: 1305 slow requests, 5 included below; oldest blocked for > 459.752194 secs 2023-07-14T11:21:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 458.749082 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 458.719956 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 458.686372 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 458.632278 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 458.578557 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: 1304 slow requests, 5 included below; oldest blocked for > 457.164608 secs 2023-07-14T11:21:39.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 456.913004 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:21:39.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 456.903125 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:21:39.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 456.893111 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:21:39.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 456.883082 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:21:39.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:38 smithi040 ceph-mon[97954]: slow request 456.872986 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:21:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: 1305 slow requests, 5 included below; oldest blocked for > 459.752194 secs 2023-07-14T11:21:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 458.749082 seconds old, received at 2023-07-14T11:13:59.105691+0000: client_request(mds.1:1622 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 458.719956 seconds old, received at 2023-07-14T11:13:59.134817+0000: client_request(mds.1:1654 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 458.686372 seconds old, received at 2023-07-14T11:13:59.168401+0000: client_request(mds.1:1686 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 458.632278 seconds old, received at 2023-07-14T11:13:59.222496+0000: client_request(mds.1:1718 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 458.578557 seconds old, received at 2023-07-14T11:13:59.276217+0000: client_request(mds.1:1750 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: 1304 slow requests, 5 included below; oldest blocked for > 457.164608 secs 2023-07-14T11:21:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 456.913004 seconds old, received at 2023-07-14T11:14:01.185160+0000: peer_request:mds.1:1639 currently dispatched 2023-07-14T11:21:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 456.903125 seconds old, received at 2023-07-14T11:14:01.195038+0000: peer_request:mds.1:1671 currently dispatched 2023-07-14T11:21:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 456.893111 seconds old, received at 2023-07-14T11:14:01.205052+0000: peer_request:mds.1:1703 currently dispatched 2023-07-14T11:21:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 456.883082 seconds old, received at 2023-07-14T11:14:01.215081+0000: peer_request:mds.1:1735 currently dispatched 2023-07-14T11:21:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:38 smithi045 ceph-mon[128791]: slow request 456.872986 seconds old, received at 2023-07-14T11:14:01.225178+0000: peer_request:mds.1:1767 currently dispatched 2023-07-14T11:21:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:39 smithi130 ceph-mon[129405]: pgmap v294: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:39 smithi040 ceph-mon[97954]: pgmap v294: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:39 smithi045 ceph-mon[128791]: pgmap v294: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:41.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:41.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:41 smithi045 ceph-mon[128791]: pgmap v295: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:41 smithi130 ceph-mon[129405]: pgmap v295: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:41 smithi040 ceph-mon[97954]: pgmap v295: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:43.814 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: pgmap v296: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:43.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: 1290 slow requests, 5 included below; oldest blocked for > 462.164703 secs 2023-07-14T11:21:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: slow request 461.863007 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:21:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: slow request 461.853005 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:21:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: slow request 461.843235 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:21:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: slow request 461.837507 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:21:43.816 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:43 smithi045 ceph-mon[128791]: slow request 461.831724 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:21:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: pgmap v296: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: 1290 slow requests, 5 included below; oldest blocked for > 462.164703 secs 2023-07-14T11:21:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: slow request 461.863007 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:21:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: slow request 461.853005 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:21:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: slow request 461.843235 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:21:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: slow request 461.837507 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:21:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:43 smithi130 ceph-mon[129405]: slow request 461.831724 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:21:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: pgmap v296: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: 1290 slow requests, 5 included below; oldest blocked for > 462.164703 secs 2023-07-14T11:21:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: slow request 461.863007 seconds old, received at 2023-07-14T11:14:01.235252+0000: peer_request:mds.1:1799 currently dispatched 2023-07-14T11:21:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: slow request 461.853005 seconds old, received at 2023-07-14T11:14:01.245253+0000: peer_request:mds.1:1831 currently dispatched 2023-07-14T11:21:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: slow request 461.843235 seconds old, received at 2023-07-14T11:14:01.255024+0000: peer_request:mds.1:1863 currently dispatched 2023-07-14T11:21:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: slow request 461.837507 seconds old, received at 2023-07-14T11:14:01.260752+0000: peer_request:mds.1:1895 currently dispatched 2023-07-14T11:21:44.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:43 smithi040 ceph-mon[97954]: slow request 461.831724 seconds old, received at 2023-07-14T11:14:01.266534+0000: peer_request:mds.1:1927 currently dispatched 2023-07-14T11:21:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: 1290 slow requests, 5 included below; oldest blocked for > 465.043510 secs 2023-07-14T11:21:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: slow request 463.843603 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: slow request 463.815324 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: slow request 463.787744 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: slow request 463.761195 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:44 smithi130 ceph-mon[129405]: slow request 463.734955 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: 1290 slow requests, 5 included below; oldest blocked for > 465.043510 secs 2023-07-14T11:21:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: slow request 463.843603 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: slow request 463.815324 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: slow request 463.787744 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: slow request 463.761195 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:44 smithi040 ceph-mon[97954]: slow request 463.734955 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: 1290 slow requests, 5 included below; oldest blocked for > 465.043510 secs 2023-07-14T11:21:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: slow request 463.843603 seconds old, received at 2023-07-14T11:13:59.302487+0000: client_request(mds.1:1782 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: slow request 463.815324 seconds old, received at 2023-07-14T11:13:59.330766+0000: client_request(mds.1:1814 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: slow request 463.787744 seconds old, received at 2023-07-14T11:13:59.358346+0000: client_request(mds.1:1846 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: slow request 463.761195 seconds old, received at 2023-07-14T11:13:59.384895+0000: client_request(mds.1:1878 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:45.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:44 smithi045 ceph-mon[128791]: slow request 463.734955 seconds old, received at 2023-07-14T11:13:59.411135+0000: client_request(mds.1:1910 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:45 smithi130 ceph-mon[129405]: pgmap v297: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:45 smithi040 ceph-mon[97954]: pgmap v297: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:45 smithi045 ceph-mon[128791]: pgmap v297: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: pgmap v298: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:21:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:21:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:47 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:21:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: pgmap v298: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:21:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:21:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:47 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:21:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: pgmap v298: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:21:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:21:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:47 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:21:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: 1274 slow requests, 5 included below; oldest blocked for > 467.164867 secs 2023-07-14T11:21:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: slow request 466.826217 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:21:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: slow request 466.820377 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:21:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: slow request 466.814518 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:21:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: slow request 466.808916 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:21:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:48 smithi130 ceph-mon[129405]: slow request 466.803206 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:21:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: 1274 slow requests, 5 included below; oldest blocked for > 467.164867 secs 2023-07-14T11:21:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: slow request 466.826217 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:21:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: slow request 466.820377 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:21:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: slow request 466.814518 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:21:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: slow request 466.808916 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:21:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:48 smithi040 ceph-mon[97954]: slow request 466.803206 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:21:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: 1274 slow requests, 5 included below; oldest blocked for > 467.164867 secs 2023-07-14T11:21:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: slow request 466.826217 seconds old, received at 2023-07-14T11:14:01.272206+0000: peer_request:mds.1:1959 currently dispatched 2023-07-14T11:21:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: slow request 466.820377 seconds old, received at 2023-07-14T11:14:01.278046+0000: peer_request:mds.1:1991 currently dispatched 2023-07-14T11:21:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: slow request 466.814518 seconds old, received at 2023-07-14T11:14:01.283906+0000: peer_request:mds.1:2023 currently dispatched 2023-07-14T11:21:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: slow request 466.808916 seconds old, received at 2023-07-14T11:14:01.289507+0000: peer_request:mds.1:2055 currently dispatched 2023-07-14T11:21:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:48 smithi045 ceph-mon[128791]: slow request 466.803206 seconds old, received at 2023-07-14T11:14:01.295217+0000: peer_request:mds.1:2087 currently dispatched 2023-07-14T11:21:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: 1274 slow requests, 5 included below; oldest blocked for > 470.068623 secs 2023-07-14T11:21:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: slow request 468.733510 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: slow request 468.707391 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: slow request 468.681178 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: slow request 468.655501 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: slow request 468.627172 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:49 smithi130 ceph-mon[129405]: pgmap v299: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: 1274 slow requests, 5 included below; oldest blocked for > 470.068623 secs 2023-07-14T11:21:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: slow request 468.733510 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: slow request 468.707391 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: slow request 468.681178 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: slow request 468.655501 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: slow request 468.627172 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:49 smithi040 ceph-mon[97954]: pgmap v299: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: 1274 slow requests, 5 included below; oldest blocked for > 470.068623 secs 2023-07-14T11:21:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: slow request 468.733510 seconds old, received at 2023-07-14T11:13:59.437693+0000: client_request(mds.1:1942 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: slow request 468.707391 seconds old, received at 2023-07-14T11:13:59.463811+0000: client_request(mds.1:1974 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: slow request 468.681178 seconds old, received at 2023-07-14T11:13:59.490024+0000: client_request(mds.1:2006 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: slow request 468.655501 seconds old, received at 2023-07-14T11:13:59.515701+0000: client_request(mds.1:2038 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: slow request 468.627172 seconds old, received at 2023-07-14T11:13:59.544030+0000: client_request(mds.1:2070 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:50.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:49 smithi045 ceph-mon[128791]: pgmap v299: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:21:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:51 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:51 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:51 smithi130 ceph-mon[129405]: pgmap v300: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:51 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:51 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:51 smithi040 ceph-mon[97954]: pgmap v300: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:51 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:51 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:52.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:51 smithi045 ceph-mon[128791]: pgmap v300: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: pgmap v301: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: 1259 slow requests, 5 included below; oldest blocked for > 472.165003 secs 2023-07-14T11:21:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: slow request 471.797570 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:21:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: slow request 471.791940 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:21:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: slow request 471.786199 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:21:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: slow request 471.780526 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:21:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:53 smithi130 ceph-mon[129405]: slow request 471.774745 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:21:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: pgmap v301: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: 1259 slow requests, 5 included below; oldest blocked for > 472.165003 secs 2023-07-14T11:21:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: slow request 471.797570 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:21:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: slow request 471.791940 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:21:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: slow request 471.786199 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:21:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: slow request 471.780526 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:21:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:53 smithi040 ceph-mon[97954]: slow request 471.774745 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:21:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: pgmap v301: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: 1259 slow requests, 5 included below; oldest blocked for > 472.165003 secs 2023-07-14T11:21:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: slow request 471.797570 seconds old, received at 2023-07-14T11:14:01.300989+0000: peer_request:mds.1:2119 currently dispatched 2023-07-14T11:21:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: slow request 471.791940 seconds old, received at 2023-07-14T11:14:01.306620+0000: peer_request:mds.1:2151 currently dispatched 2023-07-14T11:21:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: slow request 471.786199 seconds old, received at 2023-07-14T11:14:01.312360+0000: peer_request:mds.1:2183 currently dispatched 2023-07-14T11:21:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: slow request 471.780526 seconds old, received at 2023-07-14T11:14:01.318033+0000: peer_request:mds.1:2215 currently dispatched 2023-07-14T11:21:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:53 smithi045 ceph-mon[128791]: slow request 471.774745 seconds old, received at 2023-07-14T11:14:01.323815+0000: peer_request:mds.1:2247 currently dispatched 2023-07-14T11:21:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: 1259 slow requests, 5 included below; oldest blocked for > 475.177081 secs 2023-07-14T11:21:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: slow request 473.708635 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: slow request 473.681516 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: slow request 473.649794 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: slow request 473.624072 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:54 smithi130 ceph-mon[129405]: slow request 473.598825 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: 1259 slow requests, 5 included below; oldest blocked for > 475.177081 secs 2023-07-14T11:21:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: slow request 473.708635 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: slow request 473.681516 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: slow request 473.649794 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: slow request 473.624072 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:54 smithi040 ceph-mon[97954]: slow request 473.598825 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: 1259 slow requests, 5 included below; oldest blocked for > 475.177081 secs 2023-07-14T11:21:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: slow request 473.708635 seconds old, received at 2023-07-14T11:13:59.571026+0000: client_request(mds.1:2102 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: slow request 473.681516 seconds old, received at 2023-07-14T11:13:59.598145+0000: client_request(mds.1:2134 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: slow request 473.649794 seconds old, received at 2023-07-14T11:13:59.629867+0000: client_request(mds.1:2166 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: slow request 473.624072 seconds old, received at 2023-07-14T11:13:59.655589+0000: client_request(mds.1:2198 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:54 smithi045 ceph-mon[128791]: slow request 473.598825 seconds old, received at 2023-07-14T11:13:59.680836+0000: client_request(mds.1:2230 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:21:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:55 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:55 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:55 smithi130 ceph-mon[129405]: pgmap v302: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:55 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:55 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:55 smithi040 ceph-mon[97954]: pgmap v302: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:55 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:21:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:55 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:55 smithi045 ceph-mon[128791]: pgmap v302: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:21:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:57 smithi130 ceph-mon[129405]: pgmap v303: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:58.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:57 smithi040 ceph-mon[97954]: pgmap v303: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:21:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:21:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:57 smithi045 ceph-mon[128791]: pgmap v303: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:21:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: 1243 slow requests, 5 included below; oldest blocked for > 477.165088 secs 2023-07-14T11:21:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: slow request 476.768773 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:21:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: slow request 476.762760 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:21:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: slow request 476.756912 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:21:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: slow request 476.750948 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:21:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:58 smithi130 ceph-mon[129405]: slow request 476.744959 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:21:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: 1243 slow requests, 5 included below; oldest blocked for > 477.165088 secs 2023-07-14T11:21:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: slow request 476.768773 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:21:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: slow request 476.762760 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:21:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: slow request 476.756912 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:21:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: slow request 476.750948 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:21:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:58 smithi040 ceph-mon[97954]: slow request 476.744959 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:21:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: 1243 slow requests, 5 included below; oldest blocked for > 477.165088 secs 2023-07-14T11:21:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: slow request 476.768773 seconds old, received at 2023-07-14T11:14:01.329871+0000: peer_request:mds.1:2279 currently dispatched 2023-07-14T11:21:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: slow request 476.762760 seconds old, received at 2023-07-14T11:14:01.335884+0000: peer_request:mds.1:2311 currently dispatched 2023-07-14T11:21:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: slow request 476.756912 seconds old, received at 2023-07-14T11:14:01.341732+0000: peer_request:mds.1:2343 currently dispatched 2023-07-14T11:21:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: slow request 476.750948 seconds old, received at 2023-07-14T11:14:01.347696+0000: peer_request:mds.1:2375 currently dispatched 2023-07-14T11:21:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:58 smithi045 ceph-mon[128791]: slow request 476.744959 seconds old, received at 2023-07-14T11:14:01.353685+0000: peer_request:mds.1:2407 currently dispatched 2023-07-14T11:22:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: 1243 slow requests, 5 included below; oldest blocked for > 480.189921 secs 2023-07-14T11:22:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: slow request 478.586606 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: slow request 478.557260 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: slow request 478.518386 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: slow request 478.488586 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: slow request 478.462404 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:21:59 smithi130 ceph-mon[129405]: pgmap v304: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: 1243 slow requests, 5 included below; oldest blocked for > 480.189921 secs 2023-07-14T11:22:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: slow request 478.586606 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: slow request 478.557260 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: slow request 478.518386 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: slow request 478.488586 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: slow request 478.462404 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:21:59 smithi040 ceph-mon[97954]: pgmap v304: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: 1243 slow requests, 5 included below; oldest blocked for > 480.189921 secs 2023-07-14T11:22:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: slow request 478.586606 seconds old, received at 2023-07-14T11:13:59.705895+0000: client_request(mds.1:2262 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: slow request 478.557260 seconds old, received at 2023-07-14T11:13:59.735241+0000: client_request(mds.1:2294 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: slow request 478.518386 seconds old, received at 2023-07-14T11:13:59.774115+0000: client_request(mds.1:2326 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: slow request 478.488586 seconds old, received at 2023-07-14T11:13:59.803915+0000: client_request(mds.1:2358 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: slow request 478.462404 seconds old, received at 2023-07-14T11:13:59.830097+0000: client_request(mds.1:2390 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:21:59 smithi045 ceph-mon[128791]: pgmap v304: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:01.025 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:22:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:02.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:01 smithi130 ceph-mon[129405]: pgmap v305: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:02.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:01 smithi040 ceph-mon[97954]: pgmap v305: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:02.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:02.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:01 smithi045 ceph-mon[128791]: pgmap v305: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: pgmap v306: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: 1229 slow requests, 5 included below; oldest blocked for > 482.165252 secs 2023-07-14T11:22:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: slow request 481.739276 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:22:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: slow request 481.733390 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:22:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: slow request 481.727554 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:22:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: slow request 481.721658 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:22:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:03 smithi130 ceph-mon[129405]: slow request 481.715730 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:22:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: pgmap v306: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: 1229 slow requests, 5 included below; oldest blocked for > 482.165252 secs 2023-07-14T11:22:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: slow request 481.739276 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:22:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: slow request 481.733390 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:22:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: slow request 481.727554 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:22:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: slow request 481.721658 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:22:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:03 smithi040 ceph-mon[97954]: slow request 481.715730 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:22:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: pgmap v306: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: 1229 slow requests, 5 included below; oldest blocked for > 482.165252 secs 2023-07-14T11:22:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: slow request 481.739276 seconds old, received at 2023-07-14T11:14:01.359533+0000: peer_request:mds.1:2439 currently dispatched 2023-07-14T11:22:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: slow request 481.733390 seconds old, received at 2023-07-14T11:14:01.365419+0000: peer_request:mds.1:2471 currently dispatched 2023-07-14T11:22:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: slow request 481.727554 seconds old, received at 2023-07-14T11:14:01.371254+0000: peer_request:mds.1:2503 currently dispatched 2023-07-14T11:22:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: slow request 481.721658 seconds old, received at 2023-07-14T11:14:01.377150+0000: peer_request:mds.1:2535 currently dispatched 2023-07-14T11:22:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:03 smithi045 ceph-mon[128791]: slow request 481.715730 seconds old, received at 2023-07-14T11:14:01.383078+0000: peer_request:mds.1:2567 currently dispatched 2023-07-14T11:22:04.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.591+0000 7f465b39d700 1 -- 172.21.15.40:0/1469368989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 msgr2=0x7f465410c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.591+0000 7f465b39d700 1 --2- 172.21.15.40:0/1469368989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465410c810 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f464c00b0a0 tx=0x7f464c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 -- 172.21.15.40:0/1469368989 shutdown_connections 2023-07-14T11:22:04.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 --2- 172.21.15.40:0/1469368989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465410c810 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 --2- 172.21.15.40:0/1469368989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654105db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 --2- 172.21.15.40:0/1469368989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 0x7f46540ffe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 -- 172.21.15.40:0/1469368989 >> 172.21.15.40:0/1469368989 conn(0x7f46540fb4b0 msgr2=0x7f46540fd8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 -- 172.21.15.40:0/1469368989 shutdown_connections 2023-07-14T11:22:04.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.592+0000 7f465b39d700 1 -- 172.21.15.40:0/1469368989 wait complete. 2023-07-14T11:22:04.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.593+0000 7f465b39d700 1 Processor -- start 2023-07-14T11:22:04.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.593+0000 7f465b39d700 1 -- start start 2023-07-14T11:22:04.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 0x7f465419dad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465419e010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46541981b0 con 0x7f46540ff9d0 2023-07-14T11:22:04.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4654198320 con 0x7f4654100380 2023-07-14T11:22:04.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465b39d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f46541984e0 con 0x7f46541059e0 2023-07-14T11:22:04.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465ab9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465419e010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465ab9c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:53340/0 (socket says 172.21.15.40:53340) 2023-07-14T11:22:04.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465ab9c700 1 -- 172.21.15.40:0/144950818 learned_addr learned my addr 172.21.15.40:0/144950818 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:04.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465a39b700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 0x7f465419dad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465ab9c700 1 -- 172.21.15.40:0/144950818 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 msgr2=0x7f465419e010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.594+0000 7f465ab9c700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465419e010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465ab9c700 1 -- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 msgr2=0x7f465419dad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465ab9c700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 0x7f465419dad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465ab9c700 1 -- 172.21.15.40:0/144950818 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4644009520 con 0x7f46541059e0 2023-07-14T11:22:04.601 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465ab9c700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f464c009fd0 tx=0x7f464c00bce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f464b7fe700 1 -- 172.21.15.40:0/144950818 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f464c00e040 con 0x7f46541059e0 2023-07-14T11:22:04.602 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f464b7fe700 1 -- 172.21.15.40:0/144950818 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f464c004e60 con 0x7f46541059e0 2023-07-14T11:22:04.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f464b7fe700 1 -- 172.21.15.40:0/144950818 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f464c0053c0 con 0x7f46541059e0 2023-07-14T11:22:04.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f464c009d00 con 0x7f46541059e0 2023-07-14T11:22:04.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.595+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f46541a48e0 con 0x7f46541059e0 2023-07-14T11:22:04.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.600+0000 7f466ff59b80 1 Processor -- start 2023-07-14T11:22:04.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.601+0000 7f464b7fe700 1 -- 172.21.15.40:0/144950818 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f464c01da30 con 0x7f46541059e0 2023-07-14T11:22:04.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.601+0000 7f466ff59b80 1 -- start start 2023-07-14T11:22:04.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae757b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae757e110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7476cb0 con 0x557ae7571e00 2023-07-14T11:22:04.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae74c9a40 con 0x557ae757bd30 2023-07-14T11:22:04.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f466ff59b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae74c2ed0 con 0x557ae75714f0 2023-07-14T11:22:04.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:53342/0 (socket says 172.21.15.40:53342) 2023-07-14T11:22:04.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f4659b9a700 1 -- 172.21.15.40:0/26061880 learned_addr learned my addr 172.21.15.40:0/26061880 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:04.607 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f4659b9a700 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae757e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.602+0000 7f465a39b700 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae757b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 -- 172.21.15.40:0/26061880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 msgr2=0x557ae757e110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae757e110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 -- 172.21.15.40:0/26061880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 msgr2=0x557ae757b7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae757b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 -- 172.21.15.40:0/26061880 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557ae72f0de0 con 0x557ae75714f0 2023-07-14T11:22:04.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f464b7fe700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 0x7f46400a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f464b7fe700 1 -- 172.21.15.40:0/144950818 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f464c0ce6d0 con 0x7f46541059e0 2023-07-14T11:22:04.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4659b9a700 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f4644011a10 tx=0x7f4644009400 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3c1117e79c2d61fa server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f465a39b700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 0x7f46400a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4648bfe700 1 -- 172.21.15.40:0/26061880 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4644011110 con 0x557ae75714f0 2023-07-14T11:22:04.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4648bfe700 1 -- 172.21.15.40:0/26061880 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f46440044f0 con 0x557ae75714f0 2023-07-14T11:22:04.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.603+0000 7f4648bfe700 1 -- 172.21.15.40:0/26061880 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4644003d20 con 0x557ae75714f0 2023-07-14T11:22:04.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.604+0000 7f465a39b700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 0x7f46400a3890 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f4654100b10 tx=0x7f465000b500 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.604+0000 7f466ff59b80 1 -- 172.21.15.40:0/26061880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 msgr2=0x557ae75718c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.604+0000 7f466ff59b80 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7f4644011a10 tx=0x7f4644009400 comp rx=0 tx=0).stop 2023-07-14T11:22:04.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 -- 172.21.15.40:0/26061880 shutdown_connections 2023-07-14T11:22:04.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae757e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae75718c0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 --2- 172.21.15.40:0/26061880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae757b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 -- 172.21.15.40:0/26061880 >> 172.21.15.40:0/26061880 conn(0x557ae7566ff0 msgr2=0x557ae756c930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 -- 172.21.15.40:0/26061880 shutdown_connections 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 -- 172.21.15.40:0/26061880 wait complete. 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.605+0000 7f466ff59b80 1 -- start start 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.606+0000 7f466ff59b80 1 Processor -- start 2023-07-14T11:22:04.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.606+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 0x557ae757b730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.606+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae757bd30 0x557ae7579030 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f466ff59b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7462ac0 con 0x557ae75714f0 2023-07-14T11:22:04.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f466ff59b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae74627c0 con 0x557ae7571e00 2023-07-14T11:22:04.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f466ff59b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7346920 con 0x557ae757bd30 2023-07-14T11:22:04.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f466ff59b80 10 client.0 _ll_register_callbacks cb 0x557ae7460760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:22:04.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f465a39b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 0x557ae757b730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46278/0 (socket says 172.21.15.40:46278) 2023-07-14T11:22:04.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f4659b9a700 1 -- 172.21.15.40:0/3079406506 learned_addr learned my addr 172.21.15.40:0/3079406506 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:04.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f465ab9c700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae757bd30 0x557ae7579030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f4659b9a700 1 -- 172.21.15.40:0/3079406506 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae757bd30 msgr2=0x557ae7579030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.607+0000 7f4659b9a700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae757bd30 0x557ae7579030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f4659b9a700 1 -- 172.21.15.40:0/3079406506 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 msgr2=0x557ae757b730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f4659b9a700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 0x557ae757b730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f4659b9a700 1 -- 172.21.15.40:0/3079406506 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4644019040 con 0x557ae7571e00 2023-07-14T11:22:04.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f465a39b700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 0x557ae757b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:22:04.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f4659b9a700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f4644005c50 tx=0x7f4644009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=54540cb862cf73ee server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f463dffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:22:04.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4644025070 con 0x557ae7571e00 2023-07-14T11:22:04.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x557ae741c4f0 con 0x557ae7571e00 2023-07-14T11:22:04.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.608+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f4644003980 con 0x557ae7571e00 2023-07-14T11:22:04.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=171) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4644003b40 con 0x557ae7571e00 2023-07-14T11:22:04.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 10 client.26477 fetch_fsmap learned FSMap version 171 2023-07-14T11:22:04.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=171,monmap=4+}) v3 -- 0x557ae72d9c60 con 0x557ae7571e00 2023-07-14T11:22:04.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x557ae741c4f0 con 0x557ae7571e00 2023-07-14T11:22:04.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 171) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f464402a7c0 con 0x557ae7571e00 2023-07-14T11:22:04.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 10 client.26477 fetch_fsmap finished waiting for FSMap version 171 2023-07-14T11:22:04.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 10 client.26477 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:22:04.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 20 client.26477 populate_metadata read hostname 'smithi040' 2023-07-14T11:22:04.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.609+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae758b4f0 0x557ae758d900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.610+0000 7f466ff59b80 4 client.26477 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:22:04.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.610+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x557ae74bbeb0 con 0x557ae758b4f0 2023-07-14T11:22:04.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.610+0000 7f465a39b700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae758b4f0 0x557ae758d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.610+0000 7f465a39b700 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae758b4f0 0x557ae758d900 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=232227c52239f548 server_cookie=ab0c0f809d6df949 in_seq=0 out_seq=0 2023-07-14T11:22:04.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.610+0000 7f463dffb700 10 client.26477 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:04.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.611+0000 7f463dffb700 1 -- 172.21.15.40:0/3079406506 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x557ae74bbeb0 con 0x557ae758b4f0 2023-07-14T11:22:04.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.611+0000 7f463dffb700 10 client.26477 handle_command_reply: tid=0 2023-07-14T11:22:04.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.611+0000 7f466ff59b80 1 client.26477 shutdown 2023-07-14T11:22:04.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.611+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 msgr2=0x557ae75789b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.611+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f4644005c50 tx=0x7f4644009260 comp rx=0 tx=0).stop 2023-07-14T11:22:04.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 shutdown_connections 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae758b4f0 0x557ae758d900 crc :-1 s=READY pgs=109 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae757bd30 0x557ae7579030 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae75714f0 0x557ae757b730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 --2- 172.21.15.40:0/3079406506 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae75789b0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 >> 172.21.15.40:0/3079406506 conn(0x557ae7566ff0 msgr2=0x557ae756c520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.625 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f463dffb700 0 client.26477 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:04.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 shutdown_connections 2023-07-14T11:22:04.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 1 -- 172.21.15.40:0/3079406506 wait complete. 2023-07-14T11:22:04.626 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.612+0000 7f466ff59b80 20 client.26477 trim_cache size 0 max 16384 2023-07-14T11:22:04.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.643+0000 7f466ff59b80 1 Processor -- start 2023-07-14T11:22:04.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.643+0000 7f466ff59b80 1 -- start start 2023-07-14T11:22:04.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.647 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae7574270 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 0x557ae75781c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7573210 con 0x557ae757bd30 2023-07-14T11:22:04.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7572980 con 0x557ae7571e00 2023-07-14T11:22:04.648 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f466ff59b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7576f60 con 0x557ae75714f0 2023-07-14T11:22:04.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f4659b9a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae7574270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.650 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:53362/0 (socket says 172.21.15.40:53362) 2023-07-14T11:22:04.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 -- 172.21.15.40:0/146775484 learned_addr learned my addr 172.21.15.40:0/146775484 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:04.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465ab9c700 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 0x557ae75781c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 -- 172.21.15.40:0/146775484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 msgr2=0x557ae7574270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae7574270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.651 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.644+0000 7f465a39b700 1 -- 172.21.15.40:0/146775484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 msgr2=0x557ae75781c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f465a39b700 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 0x557ae75781c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f465a39b700 1 -- 172.21.15.40:0/146775484 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x557ae7317250 con 0x557ae75714f0 2023-07-14T11:22:04.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f465ab9c700 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 0x557ae75781c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:22:04.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f465a39b700 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x557ae758e1e0 tx=0x7f4650015000 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3f413d04ab15f6a1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f46493ff700 1 -- 172.21.15.40:0/146775484 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4650022040 con 0x557ae75714f0 2023-07-14T11:22:04.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f46493ff700 1 -- 172.21.15.40:0/146775484 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f465001ce50 con 0x557ae75714f0 2023-07-14T11:22:04.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.645+0000 7f46493ff700 1 -- 172.21.15.40:0/146775484 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f465001ba20 con 0x557ae75714f0 2023-07-14T11:22:04.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 -- 172.21.15.40:0/146775484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 msgr2=0x557ae7573d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.653 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x557ae758e1e0 tx=0x7f4650015000 comp rx=0 tx=0).stop 2023-07-14T11:22:04.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 -- 172.21.15.40:0/146775484 shutdown_connections 2023-07-14T11:22:04.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae757bd30 0x557ae75781c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7573d30 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 --2- 172.21.15.40:0/146775484 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae7571e00 0x557ae7574270 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.654 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.646+0000 7f466ff59b80 1 -- 172.21.15.40:0/146775484 >> 172.21.15.40:0/146775484 conn(0x557ae7566ff0 msgr2=0x557ae7587aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.647+0000 7f466ff59b80 1 -- 172.21.15.40:0/146775484 shutdown_connections 2023-07-14T11:22:04.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.647+0000 7f466ff59b80 1 -- 172.21.15.40:0/146775484 wait complete. 2023-07-14T11:22:04.655 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.647+0000 7f466ff59b80 1 -- start start 2023-07-14T11:22:04.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.647+0000 7f466ff59b80 1 Processor -- start 2023-07-14T11:22:04.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae7582740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae75807f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7580dc0 con 0x557ae7571e00 2023-07-14T11:22:04.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae7580f30 con 0x557ae757bd30 2023-07-14T11:22:04.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x557ae75810f0 con 0x557ae75714f0 2023-07-14T11:22:04.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f466ff59b80 10 client.0 _ll_register_callbacks cb 0x7f465000b3a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:22:04.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.648+0000 7f465a39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:53378/0 (socket says 172.21.15.40:53378) 2023-07-14T11:22:04.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 -- 172.21.15.40:0/643552389 learned_addr learned my addr 172.21.15.40:0/643552389 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:04.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f4659b9a700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae7582740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 -- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 msgr2=0x557ae75807f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae75807f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 -- 172.21.15.40:0/643552389 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 msgr2=0x557ae7582740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:04.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae7582740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 -- 172.21.15.40:0/643552389 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f465001c3f0 con 0x557ae75714f0 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f4659b9a700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae7582740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f465a39b700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f4650027680 tx=0x7f4650027950 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d32ec694fb270f2e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.649+0000 7f463dffb700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.653+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4650022070 con 0x557ae75714f0 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.653+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f46440044f0 con 0x557ae75714f0 2023-07-14T11:22:04.660 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.653+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(99..99 src has 1..99) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f4650005240 con 0x557ae75714f0 2023-07-14T11:22:04.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.656+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=172) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4650027e70 con 0x557ae75714f0 2023-07-14T11:22:04.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.656+0000 7f466ff59b80 10 client.26449 fetch_fsmap learned FSMap version 172 2023-07-14T11:22:04.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.656+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=172,monmap=4+}) v3 -- 0x557ae72ef2b0 con 0x557ae75714f0 2023-07-14T11:22:04.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f46440044f0 con 0x557ae75714f0 2023-07-14T11:22:04.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 172) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f4650037020 con 0x557ae75714f0 2023-07-14T11:22:04.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 10 client.26449 fetch_fsmap finished waiting for FSMap version 172 2023-07-14T11:22:04.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 10 client.26449 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:22:04.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 20 client.26449 populate_metadata read hostname 'smithi040' 2023-07-14T11:22:04.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae75e8c50 0x557ae75eb060 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:04.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 4 client.26449 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:22:04.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x557ae747e050 con 0x557ae75e8c50 2023-07-14T11:22:04.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.657+0000 7f4659b9a700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae75e8c50 0x557ae75eb060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:04.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f463dffb700 10 client.26449 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:04.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f4659b9a700 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae75e8c50 0x557ae75eb060 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7956a98cebd1ee77 server_cookie=609c2b2a573ca37b in_seq=0 out_seq=0 2023-07-14T11:22:04.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f463dffb700 1 -- 172.21.15.40:0/643552389 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x557ae747e050 con 0x557ae75e8c50 2023-07-14T11:22:04.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f463dffb700 10 client.26449 handle_command_reply: tid=0 2023-07-14T11:22:04.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f466ff59b80 1 client.26449 shutdown 2023-07-14T11:22:04.665 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 msgr2=0x557ae7582200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7f4650027680 tx=0x7f4650027950 comp rx=0 tx=0).stop 2023-07-14T11:22:04.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.658+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 shutdown_connections 2023-07-14T11:22:04.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x557ae75e8c50 0x557ae75eb060 crc :-1 s=READY pgs=110 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x557ae757bd30 0x557ae75807f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.666 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x557ae75714f0 0x557ae7582200 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f463dffb700 0 client.26449 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:04.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 --2- 172.21.15.40:0/643552389 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x557ae7571e00 0x557ae7582740 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 >> 172.21.15.40:0/643552389 conn(0x557ae7566ff0 msgr2=0x557ae7570440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 shutdown_connections 2023-07-14T11:22:04.667 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 1 -- 172.21.15.40:0/643552389 wait complete. 2023-07-14T11:22:04.668 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.659+0000 7f466ff59b80 20 client.26449 trim_cache size 0 max 16384 2023-07-14T11:22:04.668 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:22:04.668 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:22:04.668 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:22:04.669 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:22:04.669 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:22:04.669 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:22:04.669 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:22:04.669 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:22:04.670 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:22:04.670 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:22:04.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 msgr2=0x7f46400a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 0x7f46400a3890 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f4654100b10 tx=0x7f465000b500 comp rx=0 tx=0).stop 2023-07-14T11:22:04.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 msgr2=0x7f4654197b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:04.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7f464c009fd0 tx=0x7f464c00bce0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.671 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 shutdown_connections 2023-07-14T11:22:04.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f46400a13e0 0x7f46400a3890 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f46541059e0 0x7f4654197b40 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f46540ff9d0 0x7f465419dad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 --2- 172.21.15.40:0/144950818 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4654100380 0x7f465419e010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:04.672 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 >> 172.21.15.40:0/144950818 conn(0x7f46540fb4b0 msgr2=0x7f465410ab60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:04.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 shutdown_connections 2023-07-14T11:22:04.673 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:04.660+0000 7f465b39d700 1 -- 172.21.15.40:0/144950818 wait complete. 2023-07-14T11:22:04.681 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:22:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: 1229 slow requests, 5 included below; oldest blocked for > 485.201965 secs 2023-07-14T11:22:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: slow request 483.447926 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: slow request 483.422006 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: slow request 483.395866 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: slow request 483.369053 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:04 smithi130 ceph-mon[129405]: slow request 483.340701 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: 1229 slow requests, 5 included below; oldest blocked for > 485.201965 secs 2023-07-14T11:22:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: slow request 483.447926 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: slow request 483.422006 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: slow request 483.395866 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: slow request 483.369053 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:04 smithi040 ceph-mon[97954]: slow request 483.340701 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: 1229 slow requests, 5 included below; oldest blocked for > 485.201965 secs 2023-07-14T11:22:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: slow request 483.447926 seconds old, received at 2023-07-14T11:13:59.856618+0000: client_request(mds.1:2422 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: slow request 483.422006 seconds old, received at 2023-07-14T11:13:59.882538+0000: client_request(mds.1:2454 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: slow request 483.395866 seconds old, received at 2023-07-14T11:13:59.908678+0000: client_request(mds.1:2486 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: slow request 483.369053 seconds old, received at 2023-07-14T11:13:59.935491+0000: client_request(mds.1:2518 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:04 smithi045 ceph-mon[128791]: slow request 483.340701 seconds old, received at 2023-07-14T11:13:59.963843+0000: client_request(mds.1:2550 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:05 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:05 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:05 smithi130 ceph-mon[129405]: pgmap v307: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:22:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:05 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:05 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:05 smithi040 ceph-mon[97954]: pgmap v307: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:22:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:05 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:05 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:05 smithi045 ceph-mon[128791]: pgmap v307: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:22:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:07 smithi130 ceph-mon[129405]: pgmap v308: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:22:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:07 smithi040 ceph-mon[97954]: pgmap v308: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:22:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:07 smithi045 ceph-mon[128791]: pgmap v308: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:22:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: 1213 slow requests, 5 included below; oldest blocked for > 487.165357 secs 2023-07-14T11:22:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: slow request 486.709952 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:22:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: slow request 486.703945 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:22:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: slow request 486.698027 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:22:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: slow request 486.692261 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:22:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:08 smithi130 ceph-mon[129405]: slow request 486.686310 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:22:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: 1213 slow requests, 5 included below; oldest blocked for > 487.165357 secs 2023-07-14T11:22:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: slow request 486.709952 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:22:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: slow request 486.703945 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:22:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: slow request 486.698027 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:22:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: slow request 486.692261 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:22:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:08 smithi040 ceph-mon[97954]: slow request 486.686310 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:22:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: 1213 slow requests, 5 included below; oldest blocked for > 487.165357 secs 2023-07-14T11:22:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: slow request 486.709952 seconds old, received at 2023-07-14T11:14:01.388961+0000: peer_request:mds.1:2599 currently dispatched 2023-07-14T11:22:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: slow request 486.703945 seconds old, received at 2023-07-14T11:14:01.394969+0000: peer_request:mds.1:2631 currently dispatched 2023-07-14T11:22:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: slow request 486.698027 seconds old, received at 2023-07-14T11:14:01.400887+0000: peer_request:mds.1:2663 currently dispatched 2023-07-14T11:22:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: slow request 486.692261 seconds old, received at 2023-07-14T11:14:01.406653+0000: peer_request:mds.1:2695 currently dispatched 2023-07-14T11:22:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:08 smithi045 ceph-mon[128791]: slow request 486.686310 seconds old, received at 2023-07-14T11:14:01.412604+0000: peer_request:mds.1:2727 currently dispatched 2023-07-14T11:22:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: 1213 slow requests, 5 included below; oldest blocked for > 490.221406 secs 2023-07-14T11:22:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: slow request 488.332475 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: slow request 488.305060 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: slow request 488.279150 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: slow request 488.252701 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: slow request 488.225919 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: osdmap e100: 12 total, 12 up, 12 in 2023-07-14T11:22:10.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:09 smithi130 ceph-mon[129405]: pgmap v310: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:22:10.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: 1213 slow requests, 5 included below; oldest blocked for > 490.221406 secs 2023-07-14T11:22:10.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: slow request 488.332475 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: slow request 488.305060 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: slow request 488.279150 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: slow request 488.252701 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: slow request 488.225919 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: osdmap e100: 12 total, 12 up, 12 in 2023-07-14T11:22:10.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:09 smithi040 ceph-mon[97954]: pgmap v310: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:22:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: 1213 slow requests, 5 included below; oldest blocked for > 490.221406 secs 2023-07-14T11:22:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: slow request 488.332475 seconds old, received at 2023-07-14T11:13:59.991511+0000: client_request(mds.1:2582 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: slow request 488.305060 seconds old, received at 2023-07-14T11:14:00.018925+0000: client_request(mds.1:2614 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: slow request 488.279150 seconds old, received at 2023-07-14T11:14:00.044836+0000: client_request(mds.1:2646 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: slow request 488.252701 seconds old, received at 2023-07-14T11:14:00.071285+0000: client_request(mds.1:2678 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: slow request 488.225919 seconds old, received at 2023-07-14T11:14:00.098066+0000: client_request(mds.1:2710 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:10.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: osdmap e100: 12 total, 12 up, 12 in 2023-07-14T11:22:10.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:09 smithi045 ceph-mon[128791]: pgmap v310: 129 pgs: 129 active+clean; 5.9 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:22:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:11 smithi130 ceph-mon[129405]: pgmap v311: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:11 smithi040 ceph-mon[97954]: pgmap v311: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:12.060 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule status --fs=cephfs --path=/' 2023-07-14T11:22:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:11 smithi045 ceph-mon[128791]: pgmap v311: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:12.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.276+0000 7fceb7fff700 1 -- 172.21.15.40:0/2537789229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb810c540 msgr2=0x7fceb810e920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.276+0000 7fceb7fff700 1 --2- 172.21.15.40:0/2537789229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb810c540 0x7fceb810e920 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7fceb000b0a0 tx=0x7fceb000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 -- 172.21.15.40:0/2537789229 shutdown_connections 2023-07-14T11:22:12.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 --2- 172.21.15.40:0/2537789229 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb810c540 0x7fceb810e920 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 --2- 172.21.15.40:0/2537789229 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb80ff310 0x7fceb80ff6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 --2- 172.21.15.40:0/2537789229 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ffc20 0x7fceb810c000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 -- 172.21.15.40:0/2537789229 >> 172.21.15.40:0/2537789229 conn(0x7fceb8074770 msgr2=0x7fceb8074b70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:12.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 -- 172.21.15.40:0/2537789229 shutdown_connections 2023-07-14T11:22:12.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.277+0000 7fceb7fff700 1 -- 172.21.15.40:0/2537789229 wait complete. 2023-07-14T11:22:12.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.278+0000 7fceb7fff700 1 Processor -- start 2023-07-14T11:22:12.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.278+0000 7fceb7fff700 1 -- start start 2023-07-14T11:22:12.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ff310 0x7fceb8197dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 0x7fceb8198310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 0x7fceb819c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fceb819cdb0 con 0x7fceb80ff310 2023-07-14T11:22:12.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fceb819cf20 con 0x7fceb80ffc20 2023-07-14T11:22:12.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb7fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fceb819d0e0 con 0x7fceb810c540 2023-07-14T11:22:12.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 0x7fceb819c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 0x7fceb819c770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:53396/0 (socket says 172.21.15.40:53396) 2023-07-14T11:22:12.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.279+0000 7fceb77fe700 1 -- 172.21.15.40:0/3715085339 learned_addr learned my addr 172.21.15.40:0/3715085339 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:12.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 0x7fceb8198310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb6ffd700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ff310 0x7fceb8197dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 -- 172.21.15.40:0/3715085339 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 msgr2=0x7fceb819c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 0x7fceb819c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 -- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ff310 msgr2=0x7fceb8197dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ff310 0x7fceb8197dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 -- 172.21.15.40:0/3715085339 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fceac009520 con 0x7fceb80ffc20 2023-07-14T11:22:12.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.280+0000 7fceb67fc700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 0x7fceb8198310 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fceac00e9f0 tx=0x7fceac00ed00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:12.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.281+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceac00cb70 con 0x7fceb80ffc20 2023-07-14T11:22:12.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.281+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fceac00ee90 con 0x7fceb80ffc20 2023-07-14T11:22:12.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.281+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fceac003cf0 con 0x7fceb80ffc20 2023-07-14T11:22:12.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.281+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fceb0009d00 con 0x7fceb80ffc20 2023-07-14T11:22:12.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.281+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fceb81a4a20 con 0x7fceb80ffc20 2023-07-14T11:22:12.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.282+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fceb804e210 con 0x7fceb80ffc20 2023-07-14T11:22:12.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.286+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fceac00ccd0 con 0x7fceb80ffc20 2023-07-14T11:22:12.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.288+0000 7fcebc9fd700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 0x7fcea00a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.288+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 9520+0+0 (secure 0 0 0) 0x7fceac014070 con 0x7fceb80ffc20 2023-07-14T11:22:12.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.288+0000 7fceb6ffd700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 0x7fcea00a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.289+0000 7fceb6ffd700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 0x7fcea00a37f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fcea8009c60 tx=0x7fcea8009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:12.294 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.291+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fceac096860 con 0x7fceb80ffc20 2023-07-14T11:22:12.550 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.547+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}) v1 -- 0x7fceb80fcac0 con 0x7fcea00a1340 2023-07-14T11:22:12.558 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.555+0000 7fcebc9fd700 1 -- 172.21.15.40:0/3715085339 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+335 (secure 0 0 0) 0x7fceb80fcac0 con 0x7fcea00a1340 2023-07-14T11:22:12.558 INFO:teuthology.orchestra.run.smithi040.stdout:{"fs": "cephfs", "subvol": null, "path": "/", "rel_path": "/", "schedule": "1M", "retention": {"h": 3, "M": 6}, "start": "2023-07-14T00:00:00", "created": "2023-07-14T11:12:09", "first": "2023-07-14T11:13:00", "last": "2023-07-14T11:22:00", "last_pruned": "2023-07-14T11:22:01", "created_count": 10, "pruned_count": 4, "active": true} 2023-07-14T11:22:12.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 msgr2=0x7fcea00a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 0x7fcea00a37f0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fcea8009c60 tx=0x7fcea8009260 comp rx=0 tx=0).stop 2023-07-14T11:22:12.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 msgr2=0x7fceb8198310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 0x7fceb8198310 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7fceac00e9f0 tx=0x7fceac00ed00 comp rx=0 tx=0).stop 2023-07-14T11:22:12.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 shutdown_connections 2023-07-14T11:22:12.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcea00a1340 0x7fcea00a37f0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fceb810c540 0x7fceb819c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80ff310 0x7fceb8197dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 --2- 172.21.15.40:0/3715085339 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fceb80ffc20 0x7fceb8198310 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 >> 172.21.15.40:0/3715085339 conn(0x7fceb8074770 msgr2=0x7fceb80fc3b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:12.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 shutdown_connections 2023-07-14T11:22:12.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.558+0000 7fceb7fff700 1 -- 172.21.15.40:0/3715085339 wait complete. 2023-07-14T11:22:12.582 DEBUG:teuthology.orchestra.run.smithi040:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs snap-schedule list --fs=cephfs --path=/ --recursive=true' 2023-07-14T11:22:12.806 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 -- 172.21.15.40:0/449615403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 msgr2=0x7f1034105f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 --2- 172.21.15.40:0/449615403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f1034105f30 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f1024009990 tx=0x7f1024009ca0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 -- 172.21.15.40:0/449615403 shutdown_connections 2023-07-14T11:22:12.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 --2- 172.21.15.40:0/449615403 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10341004e0 0x7f103410c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 --2- 172.21.15.40:0/449615403 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f1034105f30 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 --2- 172.21.15.40:0/449615403 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10340ffb30 0x7f10340fffa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 -- 172.21.15.40:0/449615403 >> 172.21.15.40:0/449615403 conn(0x7f10340fb5f0 msgr2=0x7f10340fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:12.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 -- 172.21.15.40:0/449615403 shutdown_connections 2023-07-14T11:22:12.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.803+0000 7f103a9ff700 1 -- 172.21.15.40:0/449615403 wait complete. 2023-07-14T11:22:12.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.805+0000 7f103a9ff700 1 Processor -- start 2023-07-14T11:22:12.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.805+0000 7f103a9ff700 1 -- start start 2023-07-14T11:22:12.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10341004e0 0x7f103419c090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f10341959b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1034196020 con 0x7f10341004e0 2023-07-14T11:22:12.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1034196190 con 0x7f10340ffb30 2023-07-14T11:22:12.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f103a9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1034196350 con 0x7f1034105b60 2023-07-14T11:22:12.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f1033fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f1033fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46346/0 (socket says 172.21.15.40:46346) 2023-07-14T11:22:12.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f1033fff700 1 -- 172.21.15.40:0/1866571748 learned_addr learned my addr 172.21.15.40:0/1866571748 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:12.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f1038f9c700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f10341959b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.806+0000 7f10337fe700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10341004e0 0x7f103419c090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 -- 172.21.15.40:0/1866571748 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 msgr2=0x7f10341959b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f10341959b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 -- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10341004e0 msgr2=0x7f103419c090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:12.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10341004e0 0x7f103419c090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:12.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 -- 172.21.15.40:0/1866571748 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1028009520 con 0x7f10340ffb30 2023-07-14T11:22:12.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f1033fff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1024005910 tx=0x7f1024004710 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:12.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f102401d070 con 0x7f10340ffb30 2023-07-14T11:22:12.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1024022470 con 0x7f10340ffb30 2023-07-14T11:22:12.816 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10240095f0 con 0x7f10340ffb30 2023-07-14T11:22:12.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.807+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1024017650 con 0x7f10340ffb30 2023-07-14T11:22:12.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.808+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1034072740 con 0x7f10340ffb30 2023-07-14T11:22:12.817 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.809+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10340ff250 con 0x7f10340ffb30 2023-07-14T11:22:12.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.813+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f10240225e0 con 0x7f10340ffb30 2023-07-14T11:22:12.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.814+0000 7f10317fa700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 0x7f101c0a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:12.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.814+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(100..100 src has 1..100) v4 ==== 9520+0+0 (secure 0 0 0) 0x7f10240ce750 con 0x7f10340ffb30 2023-07-14T11:22:12.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.814+0000 7f10337fe700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 0x7f101c0a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:12.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.815+0000 7f10337fe700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 0x7f101c0a37f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f1034105400 tx=0x7f1028009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:12.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:12.815+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f10240977d0 con 0x7f10340ffb30 2023-07-14T11:22:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:13.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:13.085 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.082+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 --> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] -- mgr_command(tid 0: {"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f1034197140 con 0x7f101c0a1340 2023-07-14T11:22:13.093 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.090+0000 7f10317fa700 1 -- 172.21.15.40:0/1866571748 <== mgr.15582 v2:172.21.15.40:6800/4243839420 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+10 (secure 0 0 0) 0x7f1034197140 con 0x7f101c0a1340 2023-07-14T11:22:13.094 INFO:teuthology.orchestra.run.smithi040.stdout:/ 1M 6M3h 2023-07-14T11:22:13.096 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 msgr2=0x7f101c0a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:13.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 0x7f101c0a37f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f1034105400 tx=0x7f1028009260 comp rx=0 tx=0).stop 2023-07-14T11:22:13.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 msgr2=0x7f103419bb50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:13.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f1024005910 tx=0x7f1024004710 comp rx=0 tx=0).stop 2023-07-14T11:22:13.097 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 shutdown_connections 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f101c0a1340 0x7f101c0a37f0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1034105b60 0x7f10341959b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f10340ffb30 0x7f103419bb50 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 --2- 172.21.15.40:0/1866571748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f10341004e0 0x7f103419c090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.093+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 >> 172.21.15.40:0/1866571748 conn(0x7f10340fb5f0 msgr2=0x7f10341036c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:13.098 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.094+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 shutdown_connections 2023-07-14T11:22:13.099 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:13.094+0000 7f103a9ff700 1 -- 172.21.15.40:0/1866571748 wait complete. 2023-07-14T11:22:13.117 DEBUG:teuthology.run_tasks:Unwinding manager fwd_scrub 2023-07-14T11:22:13.129 INFO:tasks.fwd_scrub:joining ForwardScrubbers 2023-07-14T11:22:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: from='client.26486 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: pgmap v312: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: 1198 slow requests, 5 included below; oldest blocked for > 492.165423 secs 2023-07-14T11:22:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: slow request 491.680486 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:22:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: slow request 491.674625 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:22:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: slow request 491.668694 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:22:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: slow request 491.662894 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:22:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:13 smithi130 ceph-mon[129405]: slow request 491.893590 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: from='client.26486 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: pgmap v312: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: 1198 slow requests, 5 included below; oldest blocked for > 492.165423 secs 2023-07-14T11:22:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: slow request 491.680486 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:22:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: slow request 491.674625 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:22:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: slow request 491.668694 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:22:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: slow request 491.662894 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:22:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:13 smithi040 ceph-mon[97954]: slow request 491.893590 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: from='client.26486 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule status", "fs": "cephfs", "path": "/", "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: pgmap v312: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:22:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: 1198 slow requests, 5 included below; oldest blocked for > 492.165423 secs 2023-07-14T11:22:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: slow request 491.680486 seconds old, received at 2023-07-14T11:14:01.418493+0000: peer_request:mds.1:2759 currently dispatched 2023-07-14T11:22:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: slow request 491.674625 seconds old, received at 2023-07-14T11:14:01.424354+0000: peer_request:mds.1:2791 currently dispatched 2023-07-14T11:22:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: slow request 491.668694 seconds old, received at 2023-07-14T11:14:01.430285+0000: peer_request:mds.1:2823 currently dispatched 2023-07-14T11:22:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: slow request 491.662894 seconds old, received at 2023-07-14T11:14:01.436085+0000: peer_request:mds.1:2855 currently dispatched 2023-07-14T11:22:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:13 smithi045 ceph-mon[128791]: slow request 491.893590 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: from='client.26492 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:15.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: 1197 slow requests, 5 included below; oldest blocked for > 495.250009 secs 2023-07-14T11:22:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: slow request 493.228427 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: slow request 493.201642 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: slow request 493.175851 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: slow request 493.149647 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: slow request 493.121354 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:14 smithi130 ceph-mon[129405]: osdmap e101: 12 total, 12 up, 12 in 2023-07-14T11:22:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: from='client.26492 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: 1197 slow requests, 5 included below; oldest blocked for > 495.250009 secs 2023-07-14T11:22:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: slow request 493.228427 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: slow request 493.201642 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: slow request 493.175851 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: slow request 493.149647 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: slow request 493.121354 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:14 smithi040 ceph-mon[97954]: osdmap e101: 12 total, 12 up, 12 in 2023-07-14T11:22:15.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: from='client.26492 -' entity='client.admin' cmd=[{"prefix": "fs snap-schedule list", "fs": "cephfs", "path": "/", "recursive": true, "target": ["mon-mgr", ""]}]: dispatch 2023-07-14T11:22:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: 1197 slow requests, 5 included below; oldest blocked for > 495.250009 secs 2023-07-14T11:22:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: slow request 493.228427 seconds old, received at 2023-07-14T11:14:00.124162+0000: client_request(mds.1:2742 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: slow request 493.201642 seconds old, received at 2023-07-14T11:14:00.150947+0000: client_request(mds.1:2774 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: slow request 493.175851 seconds old, received at 2023-07-14T11:14:00.176738+0000: client_request(mds.1:2806 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: slow request 493.149647 seconds old, received at 2023-07-14T11:14:00.202942+0000: client_request(mds.1:2838 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: slow request 493.121354 seconds old, received at 2023-07-14T11:14:00.231235+0000: client_request(mds.1:2870 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:15.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:14 smithi045 ceph-mon[128791]: osdmap e101: 12 total, 12 up, 12 in 2023-07-14T11:22:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:15 smithi130 ceph-mon[129405]: pgmap v314: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:15 smithi040 ceph-mon[97954]: pgmap v314: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:15 smithi045 ceph-mon[128791]: pgmap v314: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:17 smithi130 ceph-mon[129405]: pgmap v315: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:17 smithi040 ceph-mon[97954]: pgmap v315: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:17 smithi045 ceph-mon[128791]: pgmap v315: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.0 KiB/s rd, 255 B/s wr, 16 op/s 2023-07-14T11:22:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: 1181 slow requests, 5 included below; oldest blocked for > 497.165513 secs 2023-07-14T11:22:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: slow request 496.893681 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: slow request 496.883668 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: slow request 496.873550 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: slow request 496.863512 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:18 smithi130 ceph-mon[129405]: slow request 496.853497 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: 1181 slow requests, 5 included below; oldest blocked for > 497.165513 secs 2023-07-14T11:22:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: slow request 496.893681 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: slow request 496.883668 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: slow request 496.873550 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: slow request 496.863512 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:18 smithi040 ceph-mon[97954]: slow request 496.853497 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: 1181 slow requests, 5 included below; oldest blocked for > 497.165513 secs 2023-07-14T11:22:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: slow request 496.893681 seconds old, received at 2023-07-14T11:14:01.205389+0000: peer_request:mds.1:1704 currently dispatched 2023-07-14T11:22:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: slow request 496.883668 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: slow request 496.873550 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: slow request 496.863512 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:18 smithi045 ceph-mon[128791]: slow request 496.853497 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: 1180 slow requests, 5 included below; oldest blocked for > 500.305829 secs 2023-07-14T11:22:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: slow request 499.184700 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: slow request 499.131259 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: slow request 499.105142 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: slow request 499.076812 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: slow request 499.049250 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:19 smithi130 ceph-mon[129405]: pgmap v316: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 204 B/s wr, 13 op/s 2023-07-14T11:22:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: 1180 slow requests, 5 included below; oldest blocked for > 500.305829 secs 2023-07-14T11:22:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: slow request 499.184700 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: slow request 499.131259 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: slow request 499.105142 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: slow request 499.076812 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: slow request 499.049250 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:19 smithi040 ceph-mon[97954]: pgmap v316: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 204 B/s wr, 13 op/s 2023-07-14T11:22:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: 1180 slow requests, 5 included below; oldest blocked for > 500.305829 secs 2023-07-14T11:22:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: slow request 499.184700 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: slow request 499.131259 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: slow request 499.105142 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: slow request 499.076812 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: slow request 499.049250 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:20.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:19 smithi045 ceph-mon[128791]: pgmap v316: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.2 KiB/s rd, 204 B/s wr, 13 op/s 2023-07-14T11:22:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:21 smithi130 ceph-mon[129405]: pgmap v317: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:22.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:22.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:22.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:21 smithi040 ceph-mon[97954]: pgmap v317: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:21 smithi045 ceph-mon[128791]: pgmap v317: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: pgmap v318: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: 1165 slow requests, 5 included below; oldest blocked for > 502.165578 secs 2023-07-14T11:22:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: slow request 501.883733 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: slow request 501.873614 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: slow request 501.863577 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: slow request 501.853561 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:23 smithi130 ceph-mon[129405]: slow request 501.843933 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: pgmap v318: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: 1165 slow requests, 5 included below; oldest blocked for > 502.165578 secs 2023-07-14T11:22:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: slow request 501.883733 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: slow request 501.873614 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: slow request 501.863577 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: slow request 501.853561 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:23 smithi040 ceph-mon[97954]: slow request 501.843933 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: pgmap v318: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.1 KiB/s rd, 102 B/s wr, 13 op/s 2023-07-14T11:22:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: 1165 slow requests, 5 included below; oldest blocked for > 502.165578 secs 2023-07-14T11:22:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: slow request 501.883733 seconds old, received at 2023-07-14T11:14:01.215401+0000: peer_request:mds.1:1736 currently dispatched 2023-07-14T11:22:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: slow request 501.873614 seconds old, received at 2023-07-14T11:14:01.225520+0000: peer_request:mds.1:1768 currently dispatched 2023-07-14T11:22:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: slow request 501.863577 seconds old, received at 2023-07-14T11:14:01.235557+0000: peer_request:mds.1:1800 currently dispatched 2023-07-14T11:22:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: slow request 501.853561 seconds old, received at 2023-07-14T11:14:01.245572+0000: peer_request:mds.1:1832 currently dispatched 2023-07-14T11:22:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:23 smithi045 ceph-mon[128791]: slow request 501.843933 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: 1164 slow requests, 5 included below; oldest blocked for > 505.427353 secs 2023-07-14T11:22:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: slow request 504.306225 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: slow request 504.252783 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: slow request 504.226667 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: slow request 504.198336 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:24 smithi130 ceph-mon[129405]: slow request 504.170775 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: 1164 slow requests, 5 included below; oldest blocked for > 505.427353 secs 2023-07-14T11:22:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: slow request 504.306225 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: slow request 504.252783 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: slow request 504.226667 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: slow request 504.198336 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:24 smithi040 ceph-mon[97954]: slow request 504.170775 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: 1164 slow requests, 5 included below; oldest blocked for > 505.427353 secs 2023-07-14T11:22:25.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: slow request 504.306225 seconds old, received at 2023-07-14T11:13:59.223708+0000: client_request(mds.1:1719 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: slow request 504.252783 seconds old, received at 2023-07-14T11:13:59.277150+0000: client_request(mds.1:1751 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: slow request 504.226667 seconds old, received at 2023-07-14T11:13:59.303266+0000: client_request(mds.1:1783 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: slow request 504.198336 seconds old, received at 2023-07-14T11:13:59.331597+0000: client_request(mds.1:1815 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:25.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:24 smithi045 ceph-mon[128791]: slow request 504.170775 seconds old, received at 2023-07-14T11:13:59.359158+0000: client_request(mds.1:1847 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:25 smithi130 ceph-mon[129405]: pgmap v319: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.5 KiB/s rd, 100 B/s wr, 17 op/s 2023-07-14T11:22:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:25 smithi040 ceph-mon[97954]: pgmap v319: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.5 KiB/s rd, 100 B/s wr, 17 op/s 2023-07-14T11:22:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:25 smithi045 ceph-mon[128791]: pgmap v319: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 9.5 KiB/s rd, 100 B/s wr, 17 op/s 2023-07-14T11:22:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:27 smithi130 ceph-mon[129405]: pgmap v320: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:27 smithi040 ceph-mon[97954]: pgmap v320: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:27 smithi045 ceph-mon[128791]: pgmap v320: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: 1149 slow requests, 5 included below; oldest blocked for > 507.165638 secs 2023-07-14T11:22:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: slow request 506.843993 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: slow request 506.838274 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: slow request 506.832491 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: slow request 506.826821 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:29.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:28 smithi130 ceph-mon[129405]: slow request 506.820972 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: 1149 slow requests, 5 included below; oldest blocked for > 507.165638 secs 2023-07-14T11:22:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: slow request 506.843993 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: slow request 506.838274 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: slow request 506.832491 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: slow request 506.826821 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:28 smithi040 ceph-mon[97954]: slow request 506.820972 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: 1149 slow requests, 5 included below; oldest blocked for > 507.165638 secs 2023-07-14T11:22:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: slow request 506.843993 seconds old, received at 2023-07-14T11:14:01.255201+0000: peer_request:mds.1:1864 currently dispatched 2023-07-14T11:22:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: slow request 506.838274 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: slow request 506.832491 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: slow request 506.826821 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:28 smithi045 ceph-mon[128791]: slow request 506.820972 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: 1147 slow requests, 5 included below; oldest blocked for > 510.510161 secs 2023-07-14T11:22:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: slow request 509.227038 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: slow request 509.200796 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: slow request 509.174256 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: slow request 509.148091 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: slow request 509.121926 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:29 smithi130 ceph-mon[129405]: pgmap v321: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: 1147 slow requests, 5 included below; oldest blocked for > 510.510161 secs 2023-07-14T11:22:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: slow request 509.227038 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: slow request 509.200796 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: slow request 509.174256 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: slow request 509.148091 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: slow request 509.121926 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:29 smithi040 ceph-mon[97954]: pgmap v321: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: 1147 slow requests, 5 included below; oldest blocked for > 510.510161 secs 2023-07-14T11:22:30.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: slow request 509.227038 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: slow request 509.200796 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: slow request 509.174256 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: slow request 509.148091 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: slow request 509.121926 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:30.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:29 smithi045 ceph-mon[128791]: pgmap v321: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:31 smithi130 ceph-mon[129405]: pgmap v322: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:31 smithi040 ceph-mon[97954]: pgmap v322: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:31 smithi045 ceph-mon[128791]: pgmap v322: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: pgmap v323: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: 1131 slow requests, 5 included below; oldest blocked for > 512.165793 secs 2023-07-14T11:22:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: slow request 511.838429 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: slow request 511.832646 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: slow request 511.826976 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: slow request 511.821128 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:33 smithi130 ceph-mon[129405]: slow request 511.815273 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: pgmap v323: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: 1131 slow requests, 5 included below; oldest blocked for > 512.165793 secs 2023-07-14T11:22:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: slow request 511.838429 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: slow request 511.832646 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: slow request 511.826976 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: slow request 511.821128 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:33 smithi040 ceph-mon[97954]: slow request 511.815273 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: pgmap v323: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: 1131 slow requests, 5 included below; oldest blocked for > 512.165793 secs 2023-07-14T11:22:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: slow request 511.838429 seconds old, received at 2023-07-14T11:14:01.260920+0000: peer_request:mds.1:1896 currently dispatched 2023-07-14T11:22:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: slow request 511.832646 seconds old, received at 2023-07-14T11:14:01.266703+0000: peer_request:mds.1:1928 currently dispatched 2023-07-14T11:22:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: slow request 511.826976 seconds old, received at 2023-07-14T11:14:01.272373+0000: peer_request:mds.1:1960 currently dispatched 2023-07-14T11:22:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: slow request 511.821128 seconds old, received at 2023-07-14T11:14:01.278221+0000: peer_request:mds.1:1992 currently dispatched 2023-07-14T11:22:34.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:33 smithi045 ceph-mon[128791]: slow request 511.815273 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:34.682 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:22:34.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 -- 172.21.15.40:0/259764766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b40728b0 msgr2=0x7f55b4072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 --2- 172.21.15.40:0/259764766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b40728b0 0x7f55b4072c80 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f55a4009990 tx=0x7f55a4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 -- 172.21.15.40:0/259764766 shutdown_connections 2023-07-14T11:22:34.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 --2- 172.21.15.40:0/259764766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b4110af0 0x7f55b4112ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 --2- 172.21.15.40:0/259764766 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b40728b0 0x7f55b4072c80 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 --2- 172.21.15.40:0/259764766 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b41105b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 -- 172.21.15.40:0/259764766 >> 172.21.15.40:0/259764766 conn(0x7f55b40fb5e0 msgr2=0x7f55b40fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 -- 172.21.15.40:0/259764766 shutdown_connections 2023-07-14T11:22:34.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.891+0000 7f55b8fff700 1 -- 172.21.15.40:0/259764766 wait complete. 2023-07-14T11:22:34.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.892+0000 7f55b8fff700 1 Processor -- start 2023-07-14T11:22:34.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 -- start start 2023-07-14T11:22:34.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b40728b0 0x7f55b419c230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4110af0 0x7f55b41a0bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b41a1210 con 0x7f55b40728b0 2023-07-14T11:22:34.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b41a1380 con 0x7f55b40731c0 2023-07-14T11:22:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b8fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f55b41a1540 con 0x7f55b4110af0 2023-07-14T11:22:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b1d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b1d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60110/0 (socket says 172.21.15.40:60110) 2023-07-14T11:22:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b1d9b700 1 -- 172.21.15.40:0/375198641 learned_addr learned my addr 172.21.15.40:0/375198641 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:34.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4110af0 0x7f55b41a0bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.893+0000 7f55b259c700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b40728b0 0x7f55b419c230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 -- 172.21.15.40:0/375198641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4110af0 msgr2=0x7f55b41a0bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4110af0 0x7f55b41a0bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 -- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b40728b0 msgr2=0x7f55b419c230 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b40728b0 0x7f55b419c230 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 -- 172.21.15.40:0/375198641 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f55a40095f0 con 0x7f55b40731c0 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b1d9b700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f55a800e9a0 tx=0x7f55a800ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55a37fe700 1 -- 172.21.15.40:0/375198641 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a8009790 con 0x7f55b40731c0 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f55b41a1820 con 0x7f55b40731c0 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55a37fe700 1 -- 172.21.15.40:0/375198641 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f55a800cbd0 con 0x7f55b40731c0 2023-07-14T11:22:34.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55a37fe700 1 -- 172.21.15.40:0/375198641 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a8018680 con 0x7f55b40731c0 2023-07-14T11:22:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.894+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55b41a9060 con 0x7f55b40731c0 2023-07-14T11:22:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.898+0000 7f55a37fe700 1 -- 172.21.15.40:0/375198641 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f55a800cd40 con 0x7f55b40731c0 2023-07-14T11:22:34.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.900+0000 7f55cc996b80 1 Processor -- start 2023-07-14T11:22:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 -- start start 2023-07-14T11:22:34.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248107be0 0x564248112590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55a37fe700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 0x7f559c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248112ad0 0x564248114eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56424800dcb0 con 0x564248107be0 2023-07-14T11:22:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x564248060a40 con 0x5642481072d0 2023-07-14T11:22:34.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55cc996b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x564248059ed0 con 0x564248112ad0 2023-07-14T11:22:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55a37fe700 1 -- 172.21.15.40:0/375198641 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f55a8014070 con 0x7f55b40731c0 2023-07-14T11:22:34.907 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55b2d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55b2d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60126/0 (socket says 172.21.15.40:60126) 2023-07-14T11:22:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.901+0000 7f55b2d9d700 1 -- 172.21.15.40:0/2951862639 learned_addr learned my addr 172.21.15.40:0/2951862639 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248112ad0 0x564248114eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 -- 172.21.15.40:0/2951862639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248112ad0 msgr2=0x564248114eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248112ad0 0x564248114eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 -- 172.21.15.40:0/2951862639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248107be0 msgr2=0x564248112590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248107be0 0x564248112590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 -- 172.21.15.40:0/2951862639 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x564247e87de0 con 0x5642481072d0 2023-07-14T11:22:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b1d9b700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 0x7f559c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.902+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f55ac013e50 tx=0x7f55ac009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bba41bf437c0ffd7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55a0ff9700 1 -- 172.21.15.40:0/2951862639 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55ac00e040 con 0x5642481072d0 2023-07-14T11:22:34.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55a0ff9700 1 -- 172.21.15.40:0/2951862639 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f55ac00f040 con 0x5642481072d0 2023-07-14T11:22:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55a0ff9700 1 -- 172.21.15.40:0/2951862639 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55ac018bb0 con 0x5642481072d0 2023-07-14T11:22:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55cc996b80 1 -- 172.21.15.40:0/2951862639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 msgr2=0x5642481076a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55b1d9b700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 0x7f559c0a3770 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f55a80a1ff0 tx=0x7f55a8099f70 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.903+0000 7f55cc996b80 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f55ac013e50 tx=0x7f55ac009920 comp rx=0 tx=0).stop 2023-07-14T11:22:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 -- 172.21.15.40:0/2951862639 shutdown_connections 2023-07-14T11:22:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248112ad0 0x564248114eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x5642481076a0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 --2- 172.21.15.40:0/2951862639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248107be0 0x564248112590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 -- 172.21.15.40:0/2951862639 >> 172.21.15.40:0/2951862639 conn(0x5642480fdda0 msgr2=0x5642481036a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 -- 172.21.15.40:0/2951862639 shutdown_connections 2023-07-14T11:22:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 -- 172.21.15.40:0/2951862639 wait complete. 2023-07-14T11:22:34.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.904+0000 7f55cc996b80 1 -- start start 2023-07-14T11:22:34.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 Processor -- start 2023-07-14T11:22:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x5642481135d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248112ad0 0x56424811df20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x564247ff9ac0 con 0x5642481072d0 2023-07-14T11:22:34.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x564247ff97c0 con 0x564248112ad0 2023-07-14T11:22:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x564247edd920 con 0x564248107be0 2023-07-14T11:22:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55cc996b80 10 client.0 _ll_register_callbacks cb 0x564247ff7760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:22:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55b2d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55b2d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45418/0 (socket says 172.21.15.40:45418) 2023-07-14T11:22:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.905+0000 7f55b2d9d700 1 -- 172.21.15.40:0/1800883108 learned_addr learned my addr 172.21.15.40:0/1800883108 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248112ad0 0x56424811df20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b259c700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x5642481135d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 -- 172.21.15.40:0/1800883108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 msgr2=0x5642481135d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x5642481135d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 -- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248112ad0 msgr2=0x56424811df20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248112ad0 0x56424811df20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 -- 172.21.15.40:0/1800883108 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55ac013b00 con 0x5642481072d0 2023-07-14T11:22:34.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b259c700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x5642481135d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:22:34.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f55ac009550 tx=0x7f55ac004c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c46e3324ac09d007 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.906+0000 7f5599ffb700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:22:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55ac00e070 con 0x5642481072d0 2023-07-14T11:22:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x564247fb34f0 con 0x5642481072d0 2023-07-14T11:22:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f55ac018d30 con 0x5642481072d0 2023-07-14T11:22:34.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=184) v2 ==== 24+0+0 (secure 0 0 0) 0x7f55ac02f6f0 con 0x5642481072d0 2023-07-14T11:22:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f55cc996b80 10 client.16452 fetch_fsmap learned FSMap version 184 2023-07-14T11:22:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=184,monmap=4+}) v3 -- 0x564247e70c60 con 0x5642481072d0 2023-07-14T11:22:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.907+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x564247fb34f0 con 0x5642481072d0 2023-07-14T11:22:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 184) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f55ac018360 con 0x5642481072d0 2023-07-14T11:22:34.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 10 client.16452 fetch_fsmap finished waiting for FSMap version 184 2023-07-14T11:22:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 10 client.16452 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:22:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 20 client.16452 populate_metadata read hostname 'smithi040' 2023-07-14T11:22:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5642481221c0 0x5642481245d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 4 client.16452 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:22:34.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x564248052eb0 con 0x5642481221c0 2023-07-14T11:22:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.908+0000 7f55b259c700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5642481221c0 0x5642481245d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.909+0000 7f5599ffb700 10 client.16452 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.909+0000 7f55b259c700 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5642481221c0 0x5642481245d0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=20a0ba4a2dca5c28 server_cookie=2731c0d4ca0f266d in_seq=0 out_seq=0 2023-07-14T11:22:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f5599ffb700 1 -- 172.21.15.40:0/1800883108 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x564248052eb0 con 0x5642481221c0 2023-07-14T11:22:34.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f5599ffb700 10 client.16452 handle_command_reply: tid=0 2023-07-14T11:22:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 client.16452 shutdown 2023-07-14T11:22:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 msgr2=0x564248113090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 crypto rx=0x7f55ac009550 tx=0x7f55ac004c10 comp rx=0 tx=0).stop 2023-07-14T11:22:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 shutdown_connections 2023-07-14T11:22:34.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5642481221c0 0x5642481245d0 crc :-1 s=READY pgs=111 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248112ad0 0x56424811df20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5642481072d0 0x564248113090 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1800883108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x5642481135d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 >> 172.21.15.40:0/1800883108 conn(0x5642480fdda0 msgr2=0x564248103290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f5599ffb700 0 client.16452 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:34.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 shutdown_connections 2023-07-14T11:22:34.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.910+0000 7f55cc996b80 1 -- 172.21.15.40:0/1800883108 wait complete. 2023-07-14T11:22:34.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.911+0000 7f55cc996b80 20 client.16452 trim_cache size 0 max 16384 2023-07-14T11:22:34.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.943+0000 7f55cc996b80 1 Processor -- start 2023-07-14T11:22:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.943+0000 7f55cc996b80 1 -- start start 2023-07-14T11:22:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.943+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x56424810d070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x564248112ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55cc996b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x564248108540 con 0x564248112ad0 2023-07-14T11:22:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55cc996b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5642480fd970 con 0x5642481072d0 2023-07-14T11:22:34.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55cc996b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x564248112390 con 0x564248107be0 2023-07-14T11:22:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b2d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x56424810d070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:36150/0 (socket says 172.21.15.40:36150) 2023-07-14T11:22:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 -- 172.21.15.40:0/1121632620 learned_addr learned my addr 172.21.15.40:0/1121632620 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x564248112ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 -- 172.21.15.40:0/1121632620 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 msgr2=0x56424810d070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x56424810d070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 -- 172.21.15.40:0/1121632620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 msgr2=0x564248112ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x564248112ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.944+0000 7f55b259c700 1 -- 172.21.15.40:0/1121632620 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5642480ff5e0 con 0x564248107be0 2023-07-14T11:22:34.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.945+0000 7f55b259c700 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f55a4015000 tx=0x7f55a4005a90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=b51c50a0577436dc server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.945+0000 7f55a17fa700 1 -- 172.21.15.40:0/1121632620 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a4026c50 con 0x564248107be0 2023-07-14T11:22:34.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.945+0000 7f55a17fa700 1 -- 172.21.15.40:0/1121632620 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f55a401e040 con 0x564248107be0 2023-07-14T11:22:34.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.945+0000 7f55a17fa700 1 -- 172.21.15.40:0/1121632620 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a4005580 con 0x564248107be0 2023-07-14T11:22:34.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 -- 172.21.15.40:0/1121632620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 msgr2=0x56424810d5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f55a4015000 tx=0x7f55a4005a90 comp rx=0 tx=0).stop 2023-07-14T11:22:34.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 -- 172.21.15.40:0/1121632620 shutdown_connections 2023-07-14T11:22:34.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x564248112ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5642481072d0 0x56424810d070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1121632620 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x564248107be0 0x56424810d5b0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.954 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.946+0000 7f55cc996b80 1 -- 172.21.15.40:0/1121632620 >> 172.21.15.40:0/1121632620 conn(0x5642480fdda0 msgr2=0x56424810c4f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.956 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.947+0000 7f55cc996b80 1 -- 172.21.15.40:0/1121632620 shutdown_connections 2023-07-14T11:22:34.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.947+0000 7f55cc996b80 1 -- 172.21.15.40:0/1121632620 wait complete. 2023-07-14T11:22:34.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 -- start start 2023-07-14T11:22:34.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 Processor -- start 2023-07-14T11:22:34.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5642481072d0 0x564248115e20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.957 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x56424816f0c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56424816f600 con 0x564248112ad0 2023-07-14T11:22:34.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56424816f770 con 0x564248107be0 2023-07-14T11:22:34.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.948+0000 7f55cc996b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56424816f8e0 con 0x5642481072d0 2023-07-14T11:22:34.958 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55cc996b80 10 client.0 _ll_register_callbacks cb 0x7f55ac004ad0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:22:34.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:60136/0 (socket says 172.21.15.40:60136) 2023-07-14T11:22:34.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 -- 172.21.15.40:0/1473017752 learned_addr learned my addr 172.21.15.40:0/1473017752 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:22:34.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5642481072d0 0x564248115e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.959 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x56424816f0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 -- 172.21.15.40:0/1473017752 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5642481072d0 msgr2=0x564248115e20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5642481072d0 0x564248115e20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 -- 172.21.15.40:0/1473017752 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 msgr2=0x56424816f0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:22:34.960 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x56424816f0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.949+0000 7f55b259c700 1 -- 172.21.15.40:0/1473017752 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f55b4113790 con 0x564248107be0 2023-07-14T11:22:34.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f55b259c700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f55a40098e0 tx=0x7f55a4026a50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b4f138cc3dcdd768 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:22:34.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f5599ffb700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:22:34.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f55a40262d0 con 0x564248107be0 2023-07-14T11:22:34.961 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f55ac00f040 con 0x564248107be0 2023-07-14T11:22:34.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f55a4026430 con 0x564248107be0 2023-07-14T11:22:34.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=184) v2 ==== 24+0+0 (secure 0 0 0) 0x7f55a40265f0 con 0x564248107be0 2023-07-14T11:22:34.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f55cc996b80 10 client.26510 fetch_fsmap learned FSMap version 184 2023-07-14T11:22:34.962 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.950+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=184,monmap=4+}) v3 -- 0x564247eae250 con 0x564248107be0 2023-07-14T11:22:34.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f55ac00f040 con 0x564248107be0 2023-07-14T11:22:34.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 184) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f55a400fa30 con 0x564248107be0 2023-07-14T11:22:34.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 10 client.26510 fetch_fsmap finished waiting for FSMap version 184 2023-07-14T11:22:34.963 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 10 client.26510 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:22:34.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 20 client.26510 populate_metadata read hostname 'smithi040' 2023-07-14T11:22:34.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564248176d60 0x564248179170 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:22:34.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 4 client.26510 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:22:34.964 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x564248015050 con 0x564248176d60 2023-07-14T11:22:34.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.951+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564248176d60 0x564248179170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:22:34.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55b2d9d700 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564248176d60 0x564248179170 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=715a2b2368255f47 server_cookie=2bdbb29819e61839 in_seq=0 out_seq=0 2023-07-14T11:22:34.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f5599ffb700 10 client.26510 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:34.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f5599ffb700 1 -- 172.21.15.40:0/1473017752 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x564248015050 con 0x564248176d60 2023-07-14T11:22:34.965 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f5599ffb700 10 client.26510 handle_command_reply: tid=0 2023-07-14T11:22:34.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 client.26510 shutdown 2023-07-14T11:22:34.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 msgr2=0x564248116360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.966 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f55a40098e0 tx=0x7f55a4026a50 comp rx=0 tx=0).stop 2023-07-14T11:22:34.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 shutdown_connections 2023-07-14T11:22:34.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x564248176d60 0x564248179170 crc :-1 s=READY pgs=112 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x564248112ad0 0x56424816f0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.967 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5642481072d0 0x564248115e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 --2- 172.21.15.40:0/1473017752 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x564248107be0 0x564248116360 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f5599ffb700 0 client.26510 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:22:34.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 >> 172.21.15.40:0/1473017752 conn(0x5642480fdda0 msgr2=0x564248112ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.968 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 shutdown_connections 2023-07-14T11:22:34.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.952+0000 7f55cc996b80 1 -- 172.21.15.40:0/1473017752 wait complete. 2023-07-14T11:22:34.969 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.953+0000 7f55cc996b80 20 client.26510 trim_cache size 0 max 16384 2023-07-14T11:22:34.969 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:22:34.970 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:22:34.970 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:22:34.970 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:22:34.971 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:22:34.971 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:22:34.971 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:22:34.971 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:22:34.971 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:22:34.972 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:22:34.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.953+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 msgr2=0x7f559c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.953+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 0x7f559c0a3770 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f55a80a1ff0 tx=0x7f55a8099f70 comp rx=0 tx=0).stop 2023-07-14T11:22:34.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.953+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 msgr2=0x7f55b419c770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:22:34.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.953+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f55a800e9a0 tx=0x7f55a800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 shutdown_connections 2023-07-14T11:22:34.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f559c0a12c0 0x7f559c0a3770 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f55b4110af0 0x7f55b41a0bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.974 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f55b40728b0 0x7f55b419c230 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 --2- 172.21.15.40:0/375198641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f55b40731c0 0x7f55b419c770 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:22:34.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 >> 172.21.15.40:0/375198641 conn(0x7f55b40fb5e0 msgr2=0x7f55b40fc240 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:22:34.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 shutdown_connections 2023-07-14T11:22:34.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:22:34.954+0000 7f55b8fff700 1 -- 172.21.15.40:0/375198641 wait complete. 2023-07-14T11:22:34.976 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: 1129 slow requests, 5 included below; oldest blocked for > 515.686369 secs 2023-07-14T11:22:34.976 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: slow request 514.403246 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:34.976 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: slow request 514.377003 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:34.976 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: slow request 514.350464 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:34.977 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: slow request 514.324299 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:34.977 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:34 smithi040 ceph-mon[97954]: slow request 514.298133 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:34.978 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:22:35.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: 1129 slow requests, 5 included below; oldest blocked for > 515.686369 secs 2023-07-14T11:22:35.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: slow request 514.403246 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: slow request 514.377003 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: slow request 514.350464 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: slow request 514.324299 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:34 smithi130 ceph-mon[129405]: slow request 514.298133 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: 1129 slow requests, 5 included below; oldest blocked for > 515.686369 secs 2023-07-14T11:22:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: slow request 514.403246 seconds old, received at 2023-07-14T11:13:59.385703+0000: client_request(mds.1:1879 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: slow request 514.377003 seconds old, received at 2023-07-14T11:13:59.411945+0000: client_request(mds.1:1911 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: slow request 514.350464 seconds old, received at 2023-07-14T11:13:59.438485+0000: client_request(mds.1:1943 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: slow request 514.324299 seconds old, received at 2023-07-14T11:13:59.464650+0000: client_request(mds.1:1975 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:35.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:34 smithi045 ceph-mon[128791]: slow request 514.298133 seconds old, received at 2023-07-14T11:13:59.490815+0000: client_request(mds.1:2007 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:36.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:35 smithi130 ceph-mon[129405]: pgmap v324: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:35 smithi040 ceph-mon[97954]: pgmap v324: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:35 smithi045 ceph-mon[128791]: pgmap v324: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:37 smithi130 ceph-mon[129405]: pgmap v325: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:37 smithi040 ceph-mon[97954]: pgmap v325: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:38.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:38.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:37 smithi045 ceph-mon[128791]: pgmap v325: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: 1115 slow requests, 5 included below; oldest blocked for > 517.165856 secs 2023-07-14T11:22:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: slow request 516.815336 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: slow request 516.809730 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: slow request 516.804015 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: slow request 516.798255 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:38 smithi130 ceph-mon[129405]: slow request 516.792625 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: 1115 slow requests, 5 included below; oldest blocked for > 517.165856 secs 2023-07-14T11:22:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: slow request 516.815336 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: slow request 516.809730 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: slow request 516.804015 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: slow request 516.798255 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:38 smithi040 ceph-mon[97954]: slow request 516.792625 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: 1115 slow requests, 5 included below; oldest blocked for > 517.165856 secs 2023-07-14T11:22:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: slow request 516.815336 seconds old, received at 2023-07-14T11:14:01.284076+0000: peer_request:mds.1:2024 currently dispatched 2023-07-14T11:22:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: slow request 516.809730 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: slow request 516.804015 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: slow request 516.798255 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:38 smithi045 ceph-mon[128791]: slow request 516.792625 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: pgmap v326: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: 1112 slow requests, 5 included below; oldest blocked for > 520.757814 secs 2023-07-14T11:22:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: slow request 519.343833 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: slow request 519.315493 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: slow request 519.288568 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: slow request 519.261225 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:39 smithi130 ceph-mon[129405]: slow request 519.229740 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: pgmap v326: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: 1112 slow requests, 5 included below; oldest blocked for > 520.757814 secs 2023-07-14T11:22:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: slow request 519.343833 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: slow request 519.315493 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: slow request 519.288568 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: slow request 519.261225 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:39 smithi040 ceph-mon[97954]: slow request 519.229740 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: pgmap v326: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: 1112 slow requests, 5 included below; oldest blocked for > 520.757814 secs 2023-07-14T11:22:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: slow request 519.343833 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: slow request 519.315493 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: slow request 519.288568 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: slow request 519.261225 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:40.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:39 smithi045 ceph-mon[128791]: slow request 519.229740 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:41 smithi130 ceph-mon[129405]: pgmap v327: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:41 smithi040 ceph-mon[97954]: pgmap v327: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:41 smithi045 ceph-mon[128791]: pgmap v327: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: pgmap v328: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: 1096 slow requests, 5 included below; oldest blocked for > 522.165989 secs 2023-07-14T11:22:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: slow request 521.809863 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: slow request 521.804148 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: slow request 521.798388 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: slow request 521.792758 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:43 smithi130 ceph-mon[129405]: slow request 521.787010 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: pgmap v328: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: 1096 slow requests, 5 included below; oldest blocked for > 522.165989 secs 2023-07-14T11:22:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: slow request 521.809863 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: slow request 521.804148 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: slow request 521.798388 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:44.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: slow request 521.792758 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:44.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:43 smithi040 ceph-mon[97954]: slow request 521.787010 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: pgmap v328: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: 1096 slow requests, 5 included below; oldest blocked for > 522.165989 secs 2023-07-14T11:22:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: slow request 521.809863 seconds old, received at 2023-07-14T11:14:01.289682+0000: peer_request:mds.1:2056 currently dispatched 2023-07-14T11:22:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: slow request 521.804148 seconds old, received at 2023-07-14T11:14:01.295398+0000: peer_request:mds.1:2088 currently dispatched 2023-07-14T11:22:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: slow request 521.798388 seconds old, received at 2023-07-14T11:14:01.301158+0000: peer_request:mds.1:2120 currently dispatched 2023-07-14T11:22:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: slow request 521.792758 seconds old, received at 2023-07-14T11:14:01.306787+0000: peer_request:mds.1:2152 currently dispatched 2023-07-14T11:22:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:43 smithi045 ceph-mon[128791]: slow request 521.787010 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: 1094 slow requests, 5 included below; oldest blocked for > 525.820511 secs 2023-07-14T11:22:45.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: slow request 524.406531 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: slow request 524.378190 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: slow request 524.351265 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: slow request 524.323922 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:44 smithi130 ceph-mon[129405]: slow request 524.292438 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: 1094 slow requests, 5 included below; oldest blocked for > 525.820511 secs 2023-07-14T11:22:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: slow request 524.406531 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: slow request 524.378190 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: slow request 524.351265 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: slow request 524.323922 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:44 smithi040 ceph-mon[97954]: slow request 524.292438 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: 1094 slow requests, 5 included below; oldest blocked for > 525.820511 secs 2023-07-14T11:22:45.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: slow request 524.406531 seconds old, received at 2023-07-14T11:13:59.516560+0000: client_request(mds.1:2039 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: slow request 524.378190 seconds old, received at 2023-07-14T11:13:59.544900+0000: client_request(mds.1:2071 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: slow request 524.351265 seconds old, received at 2023-07-14T11:13:59.571826+0000: client_request(mds.1:2103 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: slow request 524.323922 seconds old, received at 2023-07-14T11:13:59.599168+0000: client_request(mds.1:2135 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:45.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:44 smithi045 ceph-mon[128791]: slow request 524.292438 seconds old, received at 2023-07-14T11:13:59.630653+0000: client_request(mds.1:2167 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:45 smithi130 ceph-mon[129405]: pgmap v329: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:45 smithi040 ceph-mon[97954]: pgmap v329: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:45 smithi045 ceph-mon[128791]: pgmap v329: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:47 smithi130 ceph-mon[129405]: pgmap v330: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:47 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:22:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:47 smithi040 ceph-mon[97954]: pgmap v330: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:48.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:47 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:22:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:47 smithi045 ceph-mon[128791]: pgmap v330: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:47 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:22:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: 1080 slow requests, 5 included below; oldest blocked for > 527.166106 secs 2023-07-14T11:22:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: slow request 526.787127 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: slow request 526.781457 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: slow request 526.775655 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: slow request 526.769604 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:48 smithi130 ceph-mon[129405]: slow request 526.763584 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: 1080 slow requests, 5 included below; oldest blocked for > 527.166106 secs 2023-07-14T11:22:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: slow request 526.787127 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: slow request 526.781457 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: slow request 526.775655 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: slow request 526.769604 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:48 smithi040 ceph-mon[97954]: slow request 526.763584 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: 1080 slow requests, 5 included below; oldest blocked for > 527.166106 secs 2023-07-14T11:22:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: slow request 526.787127 seconds old, received at 2023-07-14T11:14:01.312536+0000: peer_request:mds.1:2184 currently dispatched 2023-07-14T11:22:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: slow request 526.781457 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: slow request 526.775655 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: slow request 526.769604 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:48 smithi045 ceph-mon[128791]: slow request 526.763584 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: pgmap v331: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: 1077 slow requests, 5 included below; oldest blocked for > 530.828895 secs 2023-07-14T11:22:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: slow request 529.275119 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: slow request 529.249845 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: slow request 529.224784 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: slow request 529.195397 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:49 smithi130 ceph-mon[129405]: slow request 529.155851 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: pgmap v331: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: 1077 slow requests, 5 included below; oldest blocked for > 530.828895 secs 2023-07-14T11:22:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: slow request 529.275119 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: slow request 529.249845 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: slow request 529.224784 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: slow request 529.195397 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:49 smithi040 ceph-mon[97954]: slow request 529.155851 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: pgmap v331: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: 1077 slow requests, 5 included below; oldest blocked for > 530.828895 secs 2023-07-14T11:22:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: slow request 529.275119 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: slow request 529.249845 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: slow request 529.224784 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: slow request 529.195397 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:50.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:49 smithi045 ceph-mon[128791]: slow request 529.155851 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:51.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:51.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:51.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:50 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:51.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:51.301 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.302 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.303 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:50 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:51.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.8", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.11", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:22:51.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:51.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:51.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:50 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.10", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:51 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:22:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:51 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:22:52.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:51 smithi130 ceph-mon[129405]: pgmap v332: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:52.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:51 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:22:52.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:51 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:22:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:51 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:22:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:51 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:22:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:51 smithi040 ceph-mon[97954]: pgmap v332: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:52.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:51 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:22:52.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:51 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:22:52.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:51 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi130 to 196.2M 2023-07-14T11:22:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:51 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi130 to 205751705: error parsing value: Value '205751705' is below minimum 939524096 2023-07-14T11:22:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:51 smithi045 ceph-mon[128791]: pgmap v332: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:51 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi045 to 197.2M 2023-07-14T11:22:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:51 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi045 to 206877081: error parsing value: Value '206877081' is below minimum 939524096 2023-07-14T11:22:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: pgmap v333: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: 1062 slow requests, 5 included below; oldest blocked for > 532.166179 secs 2023-07-14T11:22:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: slow request 531.781529 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: slow request 531.775727 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: slow request 531.769676 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: slow request 531.763656 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:53 smithi130 ceph-mon[129405]: slow request 531.757810 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: pgmap v333: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: 1062 slow requests, 5 included below; oldest blocked for > 532.166179 secs 2023-07-14T11:22:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: slow request 531.781529 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: slow request 531.775727 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: slow request 531.769676 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: slow request 531.763656 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:53 smithi040 ceph-mon[97954]: slow request 531.757810 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: pgmap v333: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: 1062 slow requests, 5 included below; oldest blocked for > 532.166179 secs 2023-07-14T11:22:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: slow request 531.781529 seconds old, received at 2023-07-14T11:14:01.318205+0000: peer_request:mds.1:2216 currently dispatched 2023-07-14T11:22:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: slow request 531.775727 seconds old, received at 2023-07-14T11:14:01.324007+0000: peer_request:mds.1:2248 currently dispatched 2023-07-14T11:22:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: slow request 531.769676 seconds old, received at 2023-07-14T11:14:01.330059+0000: peer_request:mds.1:2280 currently dispatched 2023-07-14T11:22:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: slow request 531.763656 seconds old, received at 2023-07-14T11:14:01.336078+0000: peer_request:mds.1:2312 currently dispatched 2023-07-14T11:22:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:53 smithi045 ceph-mon[128791]: slow request 531.757810 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: 1059 slow requests, 5 included below; oldest blocked for > 535.877453 secs 2023-07-14T11:22:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: slow request 534.323677 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: slow request 534.298403 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: slow request 534.273341 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: slow request 534.243955 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: slow request 534.204409 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:22:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:22:55.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:54 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: 1059 slow requests, 5 included below; oldest blocked for > 535.877453 secs 2023-07-14T11:22:55.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: slow request 534.323677 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: slow request 534.298403 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: slow request 534.273341 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: slow request 534.243955 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: slow request 534.204409 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:22:55.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:22:55.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:54 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: 1059 slow requests, 5 included below; oldest blocked for > 535.877453 secs 2023-07-14T11:22:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: slow request 534.323677 seconds old, received at 2023-07-14T11:13:59.656355+0000: client_request(mds.1:2199 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: slow request 534.298403 seconds old, received at 2023-07-14T11:13:59.681629+0000: client_request(mds.1:2231 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: slow request 534.273341 seconds old, received at 2023-07-14T11:13:59.706691+0000: client_request(mds.1:2263 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: slow request 534.243955 seconds old, received at 2023-07-14T11:13:59.736077+0000: client_request(mds.1:2295 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: slow request 534.204409 seconds old, received at 2023-07-14T11:13:59.775623+0000: client_request(mds.1:2327 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:22:55.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.9", "name": "osd_memory_target"}]: dispatch 2023-07-14T11:22:55.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:22:55.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:22:55.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:54 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:22:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:55 smithi130 ceph-mon[129405]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:22:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:55 smithi130 ceph-mon[129405]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:22:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:55 smithi130 ceph-mon[129405]: pgmap v334: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:55 smithi040 ceph-mon[97954]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:22:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:55 smithi040 ceph-mon[97954]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:22:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:55 smithi040 ceph-mon[97954]: pgmap v334: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:55 smithi045 ceph-mon[128791]: Adjusting osd_memory_target on smithi040 to 222.0M 2023-07-14T11:22:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:55 smithi045 ceph-mon[128791]: Unable to set osd_memory_target on smithi040 to 232786534: error parsing value: Value '232786534' is below minimum 939524096 2023-07-14T11:22:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:55 smithi045 ceph-mon[128791]: pgmap v334: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:22:57.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:56 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:57.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:56 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:57.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:56 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:57.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:56 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:57.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:56 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:22:57.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:56 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:22:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:57 smithi130 ceph-mon[129405]: pgmap v335: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:57 smithi040 ceph-mon[97954]: pgmap v335: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:57 smithi045 ceph-mon[128791]: pgmap v335: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:22:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: 1043 slow requests, 5 included below; oldest blocked for > 537.166310 secs 2023-07-14T11:22:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: slow request 536.757942 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: slow request 536.751999 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:22:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: slow request 536.745999 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:22:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: slow request 536.740145 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:22:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:58 smithi130 ceph-mon[129405]: slow request 536.734249 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:22:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: 1043 slow requests, 5 included below; oldest blocked for > 537.166310 secs 2023-07-14T11:22:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: slow request 536.757942 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: slow request 536.751999 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:22:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: slow request 536.745999 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:22:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: slow request 536.740145 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:22:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:58 smithi040 ceph-mon[97954]: slow request 536.734249 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:22:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: 1043 slow requests, 5 included below; oldest blocked for > 537.166310 secs 2023-07-14T11:22:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: slow request 536.757942 seconds old, received at 2023-07-14T11:14:01.341925+0000: peer_request:mds.1:2344 currently dispatched 2023-07-14T11:22:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: slow request 536.751999 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:22:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: slow request 536.745999 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:22:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: slow request 536.740145 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:22:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:58 smithi045 ceph-mon[128791]: slow request 536.734249 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:23:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: pgmap v336: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: 1040 slow requests, 5 included below; oldest blocked for > 540.931734 secs 2023-07-14T11:23:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: slow request 539.229594 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: slow request 539.203134 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: slow request 539.176908 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: slow request 539.150951 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:22:59 smithi130 ceph-mon[129405]: slow request 539.124812 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: pgmap v336: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: 1040 slow requests, 5 included below; oldest blocked for > 540.931734 secs 2023-07-14T11:23:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: slow request 539.229594 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: slow request 539.203134 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: slow request 539.176908 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: slow request 539.150951 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:22:59 smithi040 ceph-mon[97954]: slow request 539.124812 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: pgmap v336: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: 1040 slow requests, 5 included below; oldest blocked for > 540.931734 secs 2023-07-14T11:23:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: slow request 539.229594 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: slow request 539.203134 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: slow request 539.176908 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: slow request 539.150951 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:22:59 smithi045 ceph-mon[128791]: slow request 539.124812 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:01 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:01 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:02.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:01 smithi130 ceph-mon[129405]: pgmap v337: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:23:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:01 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:01 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:01 smithi040 ceph-mon[97954]: pgmap v337: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:23:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:01 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:01 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:02.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:01 smithi045 ceph-mon[128791]: pgmap v337: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:23:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: pgmap v338: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: 1024 slow requests, 5 included below; oldest blocked for > 542.166468 secs 2023-07-14T11:23:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: slow request 541.752156 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:23:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: slow request 541.746156 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:23:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: slow request 541.740302 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:23:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: slow request 541.734406 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:23:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:03 smithi130 ceph-mon[129405]: slow request 541.728601 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: pgmap v338: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: 1024 slow requests, 5 included below; oldest blocked for > 542.166468 secs 2023-07-14T11:23:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: slow request 541.752156 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:23:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: slow request 541.746156 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:23:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: slow request 541.740302 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:23:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: slow request 541.734406 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:23:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:03 smithi040 ceph-mon[97954]: slow request 541.728601 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: pgmap v338: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:23:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: 1024 slow requests, 5 included below; oldest blocked for > 542.166468 secs 2023-07-14T11:23:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: slow request 541.752156 seconds old, received at 2023-07-14T11:14:01.347867+0000: peer_request:mds.1:2376 currently dispatched 2023-07-14T11:23:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: slow request 541.746156 seconds old, received at 2023-07-14T11:14:01.353868+0000: peer_request:mds.1:2408 currently dispatched 2023-07-14T11:23:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: slow request 541.740302 seconds old, received at 2023-07-14T11:14:01.359722+0000: peer_request:mds.1:2440 currently dispatched 2023-07-14T11:23:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: slow request 541.734406 seconds old, received at 2023-07-14T11:14:01.365617+0000: peer_request:mds.1:2472 currently dispatched 2023-07-14T11:23:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:03 smithi045 ceph-mon[128791]: slow request 541.728601 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:04.980 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:23:05.203 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2740261611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 msgr2=0x7f16a410be70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2740261611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a410be70 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f1698009990 tx=0x7f1698009ca0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.204 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2740261611 shutdown_connections 2023-07-14T11:23:05.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2740261611 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 0x7f16a410e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2740261611 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a40febd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2740261611 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a410be70 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.205 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.200+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2740261611 >> 172.21.15.40:0/2740261611 conn(0x7f16a40745d0 msgr2=0x7f16a40749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.201+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2740261611 shutdown_connections 2023-07-14T11:23:05.206 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.201+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2740261611 wait complete. 2023-07-14T11:23:05.208 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 Processor -- start 2023-07-14T11:23:05.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 -- start start 2023-07-14T11:23:05.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.209 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 0x7f16a4197a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16a41980f0 con 0x7f16a410c3b0 2023-07-14T11:23:05.210 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16a4198260 con 0x7f16a40fe800 2023-07-14T11:23:05.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a8d9d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16a4198420 con 0x7f16a40ff110 2023-07-14T11:23:05.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.211 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46908/0 (socket says 172.21.15.40:46908) 2023-07-14T11:23:05.212 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a2ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:54356/0 (socket says 172.21.15.40:54356) 2023-07-14T11:23:05.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.202+0000 7f16a37fe700 1 -- 172.21.15.40:0/2601719057 learned_addr learned my addr 172.21.15.40:0/2601719057 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:05.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a3fff700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 0x7f16a4197a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.213 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 -- 172.21.15.40:0/2601719057 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 msgr2=0x7f16a419da10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.214 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 -- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 msgr2=0x7f16a4197a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 0x7f16a4197a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 -- 172.21.15.40:0/2601719057 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f16980095f0 con 0x7f16a40ff110 2023-07-14T11:23:05.215 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a37fe700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:23:05.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a2ffd700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f1698000c00 tx=0x7f1698005740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.203+0000 7f16a0ff9700 1 -- 172.21.15.40:0/2601719057 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169801d070 con 0x7f16a40ff110 2023-07-14T11:23:05.216 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.204+0000 7f16a0ff9700 1 -- 172.21.15.40:0/2601719057 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1698021410 con 0x7f16a40ff110 2023-07-14T11:23:05.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.204+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16a41986a0 con 0x7f16a40ff110 2023-07-14T11:23:05.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.204+0000 7f16a0ff9700 1 -- 172.21.15.40:0/2601719057 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1698017650 con 0x7f16a40ff110 2023-07-14T11:23:05.217 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.204+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16a41a4a00 con 0x7f16a40ff110 2023-07-14T11:23:05.223 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.207+0000 7f16a0ff9700 1 -- 172.21.15.40:0/2601719057 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f16980217f0 con 0x7f16a40ff110 2023-07-14T11:23:05.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.208+0000 7f16bd9f6b80 1 Processor -- start 2023-07-14T11:23:05.224 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.208+0000 7f16bd9f6b80 1 -- start start 2023-07-14T11:23:05.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 0x559947c3c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a0ff9700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 0x7f168c0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c3f110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.225 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947b37cb0 con 0x559947c32e00 2023-07-14T11:23:05.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947b8aa40 con 0x559947c3cd30 2023-07-14T11:23:05.226 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947b83ed0 con 0x559947c324f0 2023-07-14T11:23:05.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a0ff9700 1 -- 172.21.15.40:0/2601719057 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f16980ce1e0 con 0x7f16a40ff110 2023-07-14T11:23:05.227 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a3fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:54368/0 (socket says 172.21.15.40:54368) 2023-07-14T11:23:05.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a3fff700 1 -- 172.21.15.40:0/2194472793 learned_addr learned my addr 172.21.15.40:0/2194472793 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:05.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.209+0000 7f16a3fff700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c3f110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a2ffd700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 0x7f168c0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a37fe700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 0x559947c3c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 -- 172.21.15.40:0/2194472793 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 msgr2=0x559947c3f110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c3f110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 -- 172.21.15.40:0/2194472793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 msgr2=0x559947c3c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 0x559947c3c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 -- 172.21.15.40:0/2194472793 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5599479b1de0 con 0x559947c324f0 2023-07-14T11:23:05.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a37fe700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 0x559947c3c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:23:05.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a3fff700 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f169c008c10 tx=0x7f169c009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f3111e1a855e0e9f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f1689ffb700 1 -- 172.21.15.40:0/2194472793 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c00e050 con 0x559947c324f0 2023-07-14T11:23:05.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f1689ffb700 1 -- 172.21.15.40:0/2194472793 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f169c00f040 con 0x559947c324f0 2023-07-14T11:23:05.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f16a2ffd700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 0x7f168c0a3890 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f1698097450 tx=0x7f1698009310 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.210+0000 7f1689ffb700 1 -- 172.21.15.40:0/2194472793 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f169c018c00 con 0x559947c324f0 2023-07-14T11:23:05.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/2194472793 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 msgr2=0x559947c328c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f169c008c10 tx=0x7f169c009920 comp rx=0 tx=0).stop 2023-07-14T11:23:05.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/2194472793 shutdown_connections 2023-07-14T11:23:05.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c3f110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c324f0 0x559947c328c0 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/2194472793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c32e00 0x559947c3c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/2194472793 >> 172.21.15.40:0/2194472793 conn(0x559947c27ff0 msgr2=0x559947c2d930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/2194472793 shutdown_connections 2023-07-14T11:23:05.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.211+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/2194472793 wait complete. 2023-07-14T11:23:05.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.212+0000 7f16bd9f6b80 1 -- start start 2023-07-14T11:23:05.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.212+0000 7f16bd9f6b80 1 Processor -- start 2023-07-14T11:23:05.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c3c770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 0x559947c3a130 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947b23ac0 con 0x559947c324f0 2023-07-14T11:23:05.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947b237c0 con 0x559947c32e00 2023-07-14T11:23:05.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947a07920 con 0x559947c3cd30 2023-07-14T11:23:05.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16bd9f6b80 10 client.0 _ll_register_callbacks cb 0x559947b21760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:23:05.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:46926/0 (socket says 172.21.15.40:46926) 2023-07-14T11:23:05.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 -- 172.21.15.40:0/3823762513 learned_addr learned my addr 172.21.15.40:0/3823762513 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:05.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a3fff700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c3c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a3fff700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 0x559947c3a130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 -- 172.21.15.40:0/3823762513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 msgr2=0x559947c3a130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 0x559947c3a130 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 -- 172.21.15.40:0/3823762513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 msgr2=0x559947c3c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c3c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a37fe700 1 -- 172.21.15.40:0/3823762513 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f169c0088f0 con 0x559947c32e00 2023-07-14T11:23:05.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.213+0000 7f16a3fff700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 0x559947c3a130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:23:05.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f16a37fe700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f1694013d30 tx=0x7f16940116c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=6cf6e5cdd6f88f90 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f1682ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:23:05.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1694011ee0 con 0x559947c32e00 2023-07-14T11:23:05.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559947add4f0 con 0x559947c32e00 2023-07-14T11:23:05.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f169401b630 con 0x559947c32e00 2023-07-14T11:23:05.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=196) v2 ==== 24+0+0 (secure 0 0 0) 0x7f169401b7f0 con 0x559947c32e00 2023-07-14T11:23:05.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.214+0000 7f16bd9f6b80 10 client.26522 fetch_fsmap learned FSMap version 196 2023-07-14T11:23:05.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=196,monmap=4+}) v3 -- 0x55994799ac60 con 0x559947c32e00 2023-07-14T11:23:05.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559947add4f0 con 0x559947c32e00 2023-07-14T11:23:05.240 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 196) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f169400fb40 con 0x559947c32e00 2023-07-14T11:23:05.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 10 client.26522 fetch_fsmap finished waiting for FSMap version 196 2023-07-14T11:23:05.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 10 client.26522 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:23:05.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 20 client.26522 populate_metadata read hostname 'smithi040' 2023-07-14T11:23:05.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947c4c630 0x559947c4ea40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.241 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 4 client.26522 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:23:05.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559947b7ceb0 con 0x559947c4c630 2023-07-14T11:23:05.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.215+0000 7f16a3fff700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947c4c630 0x559947c4ea40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.216+0000 7f1682ffd700 10 client.26522 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:05.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.216+0000 7f16a3fff700 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947c4c630 0x559947c4ea40 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=e2ff8bd2e26edd53 server_cookie=f5bbe4ef2e7a89ee in_seq=0 out_seq=0 2023-07-14T11:23:05.242 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f1682ffd700 1 -- 172.21.15.40:0/3823762513 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x559947b7ceb0 con 0x559947c4c630 2023-07-14T11:23:05.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f1682ffd700 10 client.26522 handle_command_reply: tid=0 2023-07-14T11:23:05.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 client.26522 shutdown 2023-07-14T11:23:05.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 msgr2=0x559947c399f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f1694013d30 tx=0x7f16940116c0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.243 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 shutdown_connections 2023-07-14T11:23:05.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947c4c630 0x559947c4ea40 crc :-1 s=READY pgs=113 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c3cd30 0x559947c3a130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c3c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.244 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f1682ffd700 0 client.26522 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:05.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/3823762513 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c32e00 0x559947c399f0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.217+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 >> 172.21.15.40:0/3823762513 conn(0x559947c27ff0 msgr2=0x559947c2d520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.218+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 shutdown_connections 2023-07-14T11:23:05.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.218+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/3823762513 wait complete. 2023-07-14T11:23:05.245 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.218+0000 7f16bd9f6b80 20 client.26522 trim_cache size 0 max 16384 2023-07-14T11:23:05.249 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.235+0000 7f16bd9f6b80 1 Processor -- start 2023-07-14T11:23:05.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.235+0000 7f16bd9f6b80 1 -- start start 2023-07-14T11:23:05.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c34d90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c39220 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.250 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c34260 con 0x559947c324f0 2023-07-14T11:23:05.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c2e5e0 con 0x559947c3cd30 2023-07-14T11:23:05.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c49130 con 0x559947c32e00 2023-07-14T11:23:05.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c34d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:54384/0 (socket says 172.21.15.40:54384) 2023-07-14T11:23:05.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 -- 172.21.15.40:0/641185143 learned_addr learned my addr 172.21.15.40:0/641185143 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:05.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a3fff700 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c39220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 -- 172.21.15.40:0/641185143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 msgr2=0x559947c39220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c39220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 -- 172.21.15.40:0/641185143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 msgr2=0x559947c34d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c34d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a37fe700 1 -- 172.21.15.40:0/641185143 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x5599479d8250 con 0x559947c32e00 2023-07-14T11:23:05.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.236+0000 7f16a3fff700 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c39220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:23:05.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.237+0000 7f16a37fe700 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f169400e480 tx=0x7f1694011ee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3387b5fed75cd0de server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.254 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.237+0000 7f168a7fc700 1 -- 172.21.15.40:0/641185143 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1694005280 con 0x559947c32e00 2023-07-14T11:23:05.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.237+0000 7f168a7fc700 1 -- 172.21.15.40:0/641185143 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f169400e520 con 0x559947c32e00 2023-07-14T11:23:05.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.237+0000 7f168a7fc700 1 -- 172.21.15.40:0/641185143 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16940056d0 con 0x559947c32e00 2023-07-14T11:23:05.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/641185143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 msgr2=0x559947c352d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7f169400e480 tx=0x7f1694011ee0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/641185143 shutdown_connections 2023-07-14T11:23:05.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c3cd30 0x559947c39220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c324f0 0x559947c34d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/641185143 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c352d0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/641185143 >> 172.21.15.40:0/641185143 conn(0x559947c27ff0 msgr2=0x559947c48a20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/641185143 shutdown_connections 2023-07-14T11:23:05.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/641185143 wait complete. 2023-07-14T11:23:05.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.238+0000 7f16bd9f6b80 1 -- start start 2023-07-14T11:23:05.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.239+0000 7f16bd9f6b80 1 Processor -- start 2023-07-14T11:23:05.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.239+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c324f0 0x559947c43710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c3cd30 0x559947c42100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c42710 con 0x559947c3cd30 2023-07-14T11:23:05.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c42880 con 0x559947c324f0 2023-07-14T11:23:05.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559947c42a40 con 0x559947c32e00 2023-07-14T11:23:05.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16bd9f6b80 10 client.0 _ll_register_callbacks cb 0x7f1694013800 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:23:05.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c324f0 0x559947c43710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:54396/0 (socket says 172.21.15.40:54396) 2023-07-14T11:23:05.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 -- 172.21.15.40:0/588294986 learned_addr learned my addr 172.21.15.40:0/588294986 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:05.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a3fff700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c3cd30 0x559947c42100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 -- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c324f0 msgr2=0x559947c43710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c324f0 0x559947c43710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 -- 172.21.15.40:0/588294986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c3cd30 msgr2=0x559947c42100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:05.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c3cd30 0x559947c42100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.240+0000 7f16a37fe700 1 -- 172.21.15.40:0/588294986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55994799ac60 con 0x559947c32e00 2023-07-14T11:23:05.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f16a37fe700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f1694011f50 tx=0x7f1694009390 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a24aa23b1a99c9b7 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:05.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f1682ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:23:05.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1694030070 con 0x559947c32e00 2023-07-14T11:23:05.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f169c00f040 con 0x559947c32e00 2023-07-14T11:23:05.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(101..101 src has 1..101) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f1694005c80 con 0x559947c32e00 2023-07-14T11:23:05.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=196) v2 ==== 24+0+0 (secure 0 0 0) 0x7f169400ecc0 con 0x559947c32e00 2023-07-14T11:23:05.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f16bd9f6b80 10 client.26494 fetch_fsmap learned FSMap version 196 2023-07-14T11:23:05.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=196,monmap=4+}) v3 -- 0x5599479b02b0 con 0x559947c32e00 2023-07-14T11:23:05.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.241+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f169c00f040 con 0x559947c32e00 2023-07-14T11:23:05.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 196) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1694023070 con 0x559947c32e00 2023-07-14T11:23:05.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 10 client.26494 fetch_fsmap finished waiting for FSMap version 196 2023-07-14T11:23:05.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 10 client.26494 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:23:05.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 20 client.26494 populate_metadata read hostname 'smithi040' 2023-07-14T11:23:05.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947ca1790 0x559947ca3ba0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:05.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 4 client.26494 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:23:05.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x559947b3f050 con 0x559947ca1790 2023-07-14T11:23:05.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16a3fff700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947ca1790 0x559947ca3ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:05.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f16a3fff700 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947ca1790 0x559947ca3ba0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=76612e4ef2cb9182 server_cookie=91e18f0117633a78 in_seq=0 out_seq=0 2023-07-14T11:23:05.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.242+0000 7f1682ffd700 10 client.26494 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:05.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f1682ffd700 1 -- 172.21.15.40:0/588294986 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x559947b3f050 con 0x559947ca1790 2023-07-14T11:23:05.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f1682ffd700 10 client.26494 handle_command_reply: tid=0 2023-07-14T11:23:05.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 client.26494 shutdown 2023-07-14T11:23:05.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 msgr2=0x559947c41bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7f1694011f50 tx=0x7f1694009390 comp rx=0 tx=0).stop 2023-07-14T11:23:05.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 shutdown_connections 2023-07-14T11:23:05.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559947ca1790 0x559947ca3ba0 crc :-1 s=READY pgs=114 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559947c3cd30 0x559947c42100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559947c324f0 0x559947c43710 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 --2- 172.21.15.40:0/588294986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559947c32e00 0x559947c41bc0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 >> 172.21.15.40:0/588294986 conn(0x559947c27ff0 msgr2=0x559947c34d90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f1682ffd700 0 client.26494 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:05.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 shutdown_connections 2023-07-14T11:23:05.275 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 1 -- 172.21.15.40:0/588294986 wait complete. 2023-07-14T11:23:05.276 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.243+0000 7f16bd9f6b80 20 client.26494 trim_cache size 0 max 16384 2023-07-14T11:23:05.276 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:23:05.276 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:23:05.277 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:23:05.277 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:23:05.277 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:23:05.277 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:23:05.278 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:23:05.278 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:23:05.278 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:23:05.278 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:23:05.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 msgr2=0x7f168c0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 0x7f168c0a3890 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f1698097450 tx=0x7f1698009310 comp rx=0 tx=0).stop 2023-07-14T11:23:05.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 msgr2=0x7f16a419df50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:05.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f1698000c00 tx=0x7f1698005740 comp rx=0 tx=0).stop 2023-07-14T11:23:05.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 shutdown_connections 2023-07-14T11:23:05.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f168c0a13e0 0x7f168c0a3890 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f16a410c3b0 0x7f16a4197a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f16a40fe800 0x7f16a419da10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 --2- 172.21.15.40:0/2601719057 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f16a40ff110 0x7f16a419df50 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:05.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 >> 172.21.15.40:0/2601719057 conn(0x7f16a40745d0 msgr2=0x7f16a40fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:05.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 shutdown_connections 2023-07-14T11:23:05.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:05.244+0000 7f16a8d9d700 1 -- 172.21.15.40:0/2601719057 wait complete. 2023-07-14T11:23:05.285 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:23:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: 1021 slow requests, 5 included below; oldest blocked for > 545.987564 secs 2023-07-14T11:23:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: slow request 544.285425 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: slow request 544.258965 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: slow request 544.232739 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: slow request 544.206782 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: slow request 544.180643 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:05 smithi130 ceph-mon[129405]: pgmap v339: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 64 KiB/s wr, 27 op/s 2023-07-14T11:23:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: 1021 slow requests, 5 included below; oldest blocked for > 545.987564 secs 2023-07-14T11:23:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: slow request 544.285425 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: slow request 544.258965 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: slow request 544.232739 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: slow request 544.206782 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: slow request 544.180643 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:06.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:06.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:05 smithi040 ceph-mon[97954]: pgmap v339: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 64 KiB/s wr, 27 op/s 2023-07-14T11:23:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: 1021 slow requests, 5 included below; oldest blocked for > 545.987564 secs 2023-07-14T11:23:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: slow request 544.285425 seconds old, received at 2023-07-14T11:13:59.804719+0000: client_request(mds.1:2359 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: slow request 544.258965 seconds old, received at 2023-07-14T11:13:59.831179+0000: client_request(mds.1:2391 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: slow request 544.232739 seconds old, received at 2023-07-14T11:13:59.857405+0000: client_request(mds.1:2423 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: slow request 544.206782 seconds old, received at 2023-07-14T11:13:59.883362+0000: client_request(mds.1:2455 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: slow request 544.180643 seconds old, received at 2023-07-14T11:13:59.909501+0000: client_request(mds.1:2487 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:05 smithi045 ceph-mon[128791]: pgmap v339: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 54 KiB/s rd, 64 KiB/s wr, 27 op/s 2023-07-14T11:23:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:07 smithi130 ceph-mon[129405]: pgmap v340: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 64 KiB/s wr, 23 op/s 2023-07-14T11:23:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:07 smithi040 ceph-mon[97954]: pgmap v340: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 64 KiB/s wr, 23 op/s 2023-07-14T11:23:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:07 smithi045 ceph-mon[128791]: pgmap v340: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 52 KiB/s rd, 64 KiB/s wr, 23 op/s 2023-07-14T11:23:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: 1005 slow requests, 5 included below; oldest blocked for > 547.166574 secs 2023-07-14T11:23:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: slow request 546.728708 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: slow request 546.722813 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: slow request 546.716858 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: slow request 546.710982 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:08 smithi130 ceph-mon[129405]: slow request 546.704981 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: 1005 slow requests, 5 included below; oldest blocked for > 547.166574 secs 2023-07-14T11:23:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: slow request 546.728708 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: slow request 546.722813 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: slow request 546.716858 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: slow request 546.710982 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:08 smithi040 ceph-mon[97954]: slow request 546.704981 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: 1005 slow requests, 5 included below; oldest blocked for > 547.166574 secs 2023-07-14T11:23:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: slow request 546.728708 seconds old, received at 2023-07-14T11:14:01.371422+0000: peer_request:mds.1:2504 currently dispatched 2023-07-14T11:23:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: slow request 546.722813 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: slow request 546.716858 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: slow request 546.710982 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:08 smithi045 ceph-mon[128791]: slow request 546.704981 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:10.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:09 smithi130 ceph-mon[129405]: osdmap e102: 12 total, 12 up, 12 in 2023-07-14T11:23:10.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:09 smithi130 ceph-mon[129405]: pgmap v342: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 77 KiB/s wr, 27 op/s 2023-07-14T11:23:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:09 smithi040 ceph-mon[97954]: osdmap e102: 12 total, 12 up, 12 in 2023-07-14T11:23:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:09 smithi040 ceph-mon[97954]: pgmap v342: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 77 KiB/s wr, 27 op/s 2023-07-14T11:23:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:09 smithi045 ceph-mon[128791]: osdmap e102: 12 total, 12 up, 12 in 2023-07-14T11:23:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:09 smithi045 ceph-mon[128791]: pgmap v342: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 62 KiB/s rd, 77 KiB/s wr, 27 op/s 2023-07-14T11:23:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: 1001 slow requests, 5 included below; oldest blocked for > 551.055482 secs 2023-07-14T11:23:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: slow request 549.221637 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: slow request 549.193426 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: slow request 549.165660 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: slow request 549.138322 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:10 smithi130 ceph-mon[129405]: slow request 549.112379 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: 1001 slow requests, 5 included below; oldest blocked for > 551.055482 secs 2023-07-14T11:23:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: slow request 549.221637 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: slow request 549.193426 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: slow request 549.165660 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: slow request 549.138322 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:10 smithi040 ceph-mon[97954]: slow request 549.112379 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: 1001 slow requests, 5 included below; oldest blocked for > 551.055482 secs 2023-07-14T11:23:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: slow request 549.221637 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: slow request 549.193426 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: slow request 549.165660 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: slow request 549.138322 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:10 smithi045 ceph-mon[128791]: slow request 549.112379 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:11 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:11 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:12.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:11 smithi130 ceph-mon[129405]: pgmap v343: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:11 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:11 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:11 smithi040 ceph-mon[97954]: pgmap v343: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:11 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:11 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:12.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:11 smithi045 ceph-mon[128791]: pgmap v343: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: pgmap v344: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: 985 slow requests, 5 included below; oldest blocked for > 552.166669 secs 2023-07-14T11:23:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: slow request 551.722907 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: slow request 551.716951 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: slow request 551.711076 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: slow request 551.705075 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:13 smithi130 ceph-mon[129405]: slow request 551.699166 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: pgmap v344: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: 985 slow requests, 5 included below; oldest blocked for > 552.166669 secs 2023-07-14T11:23:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: slow request 551.722907 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: slow request 551.716951 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: slow request 551.711076 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: slow request 551.705075 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:13 smithi040 ceph-mon[97954]: slow request 551.699166 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: pgmap v344: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 170 KiB/s rd, 92 KiB/s wr, 28 op/s 2023-07-14T11:23:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: 985 slow requests, 5 included below; oldest blocked for > 552.166669 secs 2023-07-14T11:23:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: slow request 551.722907 seconds old, received at 2023-07-14T11:14:01.377318+0000: peer_request:mds.1:2536 currently dispatched 2023-07-14T11:23:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: slow request 551.716951 seconds old, received at 2023-07-14T11:14:01.383273+0000: peer_request:mds.1:2568 currently dispatched 2023-07-14T11:23:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: slow request 551.711076 seconds old, received at 2023-07-14T11:14:01.389149+0000: peer_request:mds.1:2600 currently dispatched 2023-07-14T11:23:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: slow request 551.705075 seconds old, received at 2023-07-14T11:14:01.395150+0000: peer_request:mds.1:2632 currently dispatched 2023-07-14T11:23:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:13 smithi045 ceph-mon[128791]: slow request 551.699166 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: 980 slow requests, 5 included below; oldest blocked for > 556.134949 secs 2023-07-14T11:23:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: slow request 554.301104 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: slow request 554.272893 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: slow request 554.245127 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: slow request 554.217788 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: slow request 554.191846 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: osdmap e103: 12 total, 12 up, 12 in 2023-07-14T11:23:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:15 smithi130 ceph-mon[129405]: pgmap v346: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 189 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: 980 slow requests, 5 included below; oldest blocked for > 556.134949 secs 2023-07-14T11:23:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: slow request 554.301104 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: slow request 554.272893 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: slow request 554.245127 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: slow request 554.217788 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: slow request 554.191846 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: osdmap e103: 12 total, 12 up, 12 in 2023-07-14T11:23:16.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:15 smithi040 ceph-mon[97954]: pgmap v346: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 189 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: 980 slow requests, 5 included below; oldest blocked for > 556.134949 secs 2023-07-14T11:23:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: slow request 554.301104 seconds old, received at 2023-07-14T11:13:59.936425+0000: client_request(mds.1:2519 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: slow request 554.272893 seconds old, received at 2023-07-14T11:13:59.964636+0000: client_request(mds.1:2551 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: slow request 554.245127 seconds old, received at 2023-07-14T11:13:59.992401+0000: client_request(mds.1:2583 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: slow request 554.217788 seconds old, received at 2023-07-14T11:14:00.019740+0000: client_request(mds.1:2615 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: slow request 554.191846 seconds old, received at 2023-07-14T11:14:00.045682+0000: client_request(mds.1:2647 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: osdmap e103: 12 total, 12 up, 12 in 2023-07-14T11:23:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:15 smithi045 ceph-mon[128791]: pgmap v346: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 189 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:17 smithi130 ceph-mon[129405]: pgmap v347: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 186 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:18.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:17 smithi040 ceph-mon[97954]: pgmap v347: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 186 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:17 smithi045 ceph-mon[128791]: pgmap v347: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 186 KiB/s rd, 66 KiB/s wr, 14 op/s 2023-07-14T11:23:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: 964 slow requests, 5 included below; oldest blocked for > 557.166793 secs 2023-07-14T11:23:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: slow request 556.699290 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: slow request 556.693525 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: slow request 556.687532 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: slow request 556.681685 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:18 smithi130 ceph-mon[129405]: slow request 556.675792 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: 964 slow requests, 5 included below; oldest blocked for > 557.166793 secs 2023-07-14T11:23:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: slow request 556.699290 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: slow request 556.693525 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: slow request 556.687532 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: slow request 556.681685 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:18 smithi040 ceph-mon[97954]: slow request 556.675792 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: 964 slow requests, 5 included below; oldest blocked for > 557.166793 secs 2023-07-14T11:23:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: slow request 556.699290 seconds old, received at 2023-07-14T11:14:01.401059+0000: peer_request:mds.1:2664 currently dispatched 2023-07-14T11:23:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: slow request 556.693525 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: slow request 556.687532 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: slow request 556.681685 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:18 smithi045 ceph-mon[128791]: slow request 556.675792 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:19 smithi130 ceph-mon[129405]: pgmap v348: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 53 KiB/s wr, 11 op/s 2023-07-14T11:23:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:20.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:19 smithi040 ceph-mon[97954]: pgmap v348: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 53 KiB/s wr, 11 op/s 2023-07-14T11:23:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:19 smithi045 ceph-mon[128791]: pgmap v348: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 151 KiB/s rd, 53 KiB/s wr, 11 op/s 2023-07-14T11:23:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: 959 slow requests, 5 included below; oldest blocked for > 561.233208 secs 2023-07-14T11:23:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: slow request 559.263693 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: slow request 559.236864 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: slow request 559.210768 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: slow request 559.183973 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:20 smithi130 ceph-mon[129405]: slow request 559.158266 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: 959 slow requests, 5 included below; oldest blocked for > 561.233208 secs 2023-07-14T11:23:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: slow request 559.263693 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: slow request 559.236864 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: slow request 559.210768 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: slow request 559.183973 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:20 smithi040 ceph-mon[97954]: slow request 559.158266 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: 959 slow requests, 5 included below; oldest blocked for > 561.233208 secs 2023-07-14T11:23:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: slow request 559.263693 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: slow request 559.236864 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: slow request 559.210768 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: slow request 559.183973 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:21.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:20 smithi045 ceph-mon[128791]: slow request 559.158266 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:21 smithi130 ceph-mon[129405]: pgmap v349: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:22.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:21 smithi040 ceph-mon[97954]: pgmap v349: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:21 smithi045 ceph-mon[128791]: pgmap v349: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: pgmap v350: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: 944 slow requests, 5 included below; oldest blocked for > 562.166884 secs 2023-07-14T11:23:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: slow request 561.693617 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: slow request 561.687624 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: slow request 561.681776 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: slow request 561.675884 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:23 smithi130 ceph-mon[129405]: slow request 561.669957 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: pgmap v350: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: 944 slow requests, 5 included below; oldest blocked for > 562.166884 secs 2023-07-14T11:23:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: slow request 561.693617 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: slow request 561.687624 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: slow request 561.681776 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: slow request 561.675884 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:23 smithi040 ceph-mon[97954]: slow request 561.669957 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: pgmap v350: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 43 KiB/s rd, 38 KiB/s wr, 11 op/s 2023-07-14T11:23:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: 944 slow requests, 5 included below; oldest blocked for > 562.166884 secs 2023-07-14T11:23:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: slow request 561.693617 seconds old, received at 2023-07-14T11:14:01.406823+0000: peer_request:mds.1:2696 currently dispatched 2023-07-14T11:23:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: slow request 561.687624 seconds old, received at 2023-07-14T11:14:01.412816+0000: peer_request:mds.1:2728 currently dispatched 2023-07-14T11:23:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: slow request 561.681776 seconds old, received at 2023-07-14T11:14:01.418664+0000: peer_request:mds.1:2760 currently dispatched 2023-07-14T11:23:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: slow request 561.675884 seconds old, received at 2023-07-14T11:14:01.424556+0000: peer_request:mds.1:2792 currently dispatched 2023-07-14T11:23:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:23 smithi045 ceph-mon[128791]: slow request 561.669957 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: 938 slow requests, 5 included below; oldest blocked for > 566.336111 secs 2023-07-14T11:23:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: slow request 564.366595 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: slow request 564.339766 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: slow request 564.313671 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: slow request 564.286876 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: slow request 564.261168 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:26.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:26.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:25 smithi130 ceph-mon[129405]: pgmap v351: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:23:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: 938 slow requests, 5 included below; oldest blocked for > 566.336111 secs 2023-07-14T11:23:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: slow request 564.366595 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: slow request 564.339766 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: slow request 564.313671 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: slow request 564.286876 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: slow request 564.261168 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:26.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:26.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:25 smithi040 ceph-mon[97954]: pgmap v351: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:23:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: 938 slow requests, 5 included below; oldest blocked for > 566.336111 secs 2023-07-14T11:23:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: slow request 564.366595 seconds old, received at 2023-07-14T11:14:00.072095+0000: client_request(mds.1:2679 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: slow request 564.339766 seconds old, received at 2023-07-14T11:14:00.098924+0000: client_request(mds.1:2711 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: slow request 564.313671 seconds old, received at 2023-07-14T11:14:00.125019+0000: client_request(mds.1:2743 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: slow request 564.286876 seconds old, received at 2023-07-14T11:14:00.151815+0000: client_request(mds.1:2775 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: slow request 564.261168 seconds old, received at 2023-07-14T11:14:00.177522+0000: client_request(mds.1:2807 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:25 smithi045 ceph-mon[128791]: pgmap v351: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 45 KiB/s rd, 38 KiB/s wr, 16 op/s 2023-07-14T11:23:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:27 smithi130 ceph-mon[129405]: pgmap v352: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:27 smithi040 ceph-mon[97954]: pgmap v352: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:27 smithi045 ceph-mon[128791]: pgmap v352: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: 924 slow requests, 5 included below; oldest blocked for > 567.166944 secs 2023-07-14T11:23:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: slow request 566.670018 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: slow request 566.664246 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: slow request 566.827930 seconds old, received at 2023-07-14T11:14:01.272570+0000: peer_request:mds.1:1961 currently dispatched 2023-07-14T11:23:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: slow request 566.822090 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:28 smithi130 ceph-mon[129405]: slow request 566.816256 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: 924 slow requests, 5 included below; oldest blocked for > 567.166944 secs 2023-07-14T11:23:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: slow request 566.670018 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: slow request 566.664246 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: slow request 566.827930 seconds old, received at 2023-07-14T11:14:01.272570+0000: peer_request:mds.1:1961 currently dispatched 2023-07-14T11:23:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: slow request 566.822090 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:28 smithi040 ceph-mon[97954]: slow request 566.816256 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: 924 slow requests, 5 included below; oldest blocked for > 567.166944 secs 2023-07-14T11:23:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: slow request 566.670018 seconds old, received at 2023-07-14T11:14:01.430482+0000: peer_request:mds.1:2824 currently dispatched 2023-07-14T11:23:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: slow request 566.664246 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: slow request 566.827930 seconds old, received at 2023-07-14T11:14:01.272570+0000: peer_request:mds.1:1961 currently dispatched 2023-07-14T11:23:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: slow request 566.822090 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:29.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:28 smithi045 ceph-mon[128791]: slow request 566.816256 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:29 smithi130 ceph-mon[129405]: pgmap v353: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:29 smithi040 ceph-mon[97954]: pgmap v353: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:29 smithi045 ceph-mon[128791]: pgmap v353: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:31.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: 918 slow requests, 5 included below; oldest blocked for > 571.391183 secs 2023-07-14T11:23:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: slow request 569.290046 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: slow request 569.261332 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: slow request 570.028262 seconds old, received at 2023-07-14T11:13:59.465500+0000: client_request(mds.1:1976 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: slow request 570.002135 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:30 smithi130 ceph-mon[129405]: slow request 569.976230 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: 918 slow requests, 5 included below; oldest blocked for > 571.391183 secs 2023-07-14T11:23:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: slow request 569.290046 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: slow request 569.261332 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: slow request 570.028262 seconds old, received at 2023-07-14T11:13:59.465500+0000: client_request(mds.1:1976 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: slow request 570.002135 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:30 smithi040 ceph-mon[97954]: slow request 569.976230 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: 918 slow requests, 5 included below; oldest blocked for > 571.391183 secs 2023-07-14T11:23:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: slow request 569.290046 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: slow request 569.261332 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: slow request 570.028262 seconds old, received at 2023-07-14T11:13:59.465500+0000: client_request(mds.1:1976 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: slow request 570.002135 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:31.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:30 smithi045 ceph-mon[128791]: slow request 569.976230 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:31 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:31 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:31 smithi130 ceph-mon[129405]: pgmap v354: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:31 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:31 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:31 smithi040 ceph-mon[97954]: pgmap v354: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:31 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:31 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:31 smithi045 ceph-mon[128791]: pgmap v354: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:34.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: pgmap v355: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: 903 slow requests, 5 included below; oldest blocked for > 572.167072 secs 2023-07-14T11:23:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: slow request 571.664374 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:34.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: slow request 571.822218 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: slow request 571.816383 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: slow request 571.810755 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:34.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:33 smithi130 ceph-mon[129405]: slow request 571.805042 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:34.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: pgmap v355: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: 903 slow requests, 5 included below; oldest blocked for > 572.167072 secs 2023-07-14T11:23:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: slow request 571.664374 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: slow request 571.822218 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: slow request 571.816383 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:34.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: slow request 571.810755 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:34.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:33 smithi040 ceph-mon[97954]: slow request 571.805042 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: pgmap v355: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: 903 slow requests, 5 included below; oldest blocked for > 572.167072 secs 2023-07-14T11:23:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: slow request 571.664374 seconds old, received at 2023-07-14T11:14:01.436254+0000: peer_request:mds.1:2856 currently dispatched 2023-07-14T11:23:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: slow request 571.822218 seconds old, received at 2023-07-14T11:14:01.278410+0000: peer_request:mds.1:1993 currently dispatched 2023-07-14T11:23:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: slow request 571.816383 seconds old, received at 2023-07-14T11:14:01.284245+0000: peer_request:mds.1:2025 currently dispatched 2023-07-14T11:23:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: slow request 571.810755 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:33 smithi045 ceph-mon[128791]: slow request 571.805042 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:35.286 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:23:35.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 -- 172.21.15.40:0/2476381808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 msgr2=0x7fcc580febd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.519 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/2476381808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc580febd0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7fcc54009990 tx=0x7fcc54009ca0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 -- 172.21.15.40:0/2476381808 shutdown_connections 2023-07-14T11:23:35.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/2476381808 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc5810c3b0 0x7fcc5810e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/2476381808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc580febd0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.520 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/2476381808 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc580ff110 0x7fcc5810be70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 -- 172.21.15.40:0/2476381808 >> 172.21.15.40:0/2476381808 conn(0x7fcc580745d0 msgr2=0x7fcc580749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 -- 172.21.15.40:0/2476381808 shutdown_connections 2023-07-14T11:23:35.521 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.515+0000 7fcc5e99d700 1 -- 172.21.15.40:0/2476381808 wait complete. 2023-07-14T11:23:35.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.516+0000 7fcc5e99d700 1 Processor -- start 2023-07-14T11:23:35.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 -- start start 2023-07-14T11:23:35.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc5819d980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.523 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc5810c3b0 0x7fcc581979f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc58198060 con 0x7fcc5810c3b0 2023-07-14T11:23:35.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc581981d0 con 0x7fcc580ff110 2023-07-14T11:23:35.524 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e99d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc58198390 con 0x7fcc580fe800 2023-07-14T11:23:35.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38482/0 (socket says 172.21.15.40:38482) 2023-07-14T11:23:35.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5d19a700 1 -- 172.21.15.40:0/1336855080 learned_addr learned my addr 172.21.15.40:0/1336855080 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:35.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc5819d980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.525 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.517+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc5810c3b0 0x7fcc581979f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 -- 172.21.15.40:0/1336855080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 msgr2=0x7fcc5819d980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc5819d980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 -- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc5810c3b0 msgr2=0x7fcc581979f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc5810c3b0 0x7fcc581979f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.526 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 -- 172.21.15.40:0/1336855080 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc540095f0 con 0x7fcc580ff110 2023-07-14T11:23:35.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fcc4800e970 tx=0x7fcc4800ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc4effd700 1 -- 172.21.15.40:0/1336855080 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc48009790 con 0x7fcc580ff110 2023-07-14T11:23:35.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc58198670 con 0x7fcc580ff110 2023-07-14T11:23:35.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc4effd700 1 -- 172.21.15.40:0/1336855080 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcc4800c7e0 con 0x7fcc580ff110 2023-07-14T11:23:35.527 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.518+0000 7fcc4effd700 1 -- 172.21.15.40:0/1336855080 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc48003c70 con 0x7fcc580ff110 2023-07-14T11:23:35.528 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.519+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc581a4970 con 0x7fcc580ff110 2023-07-14T11:23:35.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.523+0000 7fcc4effd700 1 -- 172.21.15.40:0/1336855080 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fcc4800cc50 con 0x7fcc580ff110 2023-07-14T11:23:35.529 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.523+0000 7fcc735bab80 1 Processor -- start 2023-07-14T11:23:35.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 -- start start 2023-07-14T11:23:35.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e3fc7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e3ff110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.530 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e2f7cb0 con 0x56073e3f2e00 2023-07-14T11:23:35.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e34aa40 con 0x56073e3fcd30 2023-07-14T11:23:35.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc735bab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e343ed0 con 0x56073e3f24f0 2023-07-14T11:23:35.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc5e19c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e3ff110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.531 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc5e19c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58508/0 (socket says 172.21.15.40:58508) 2023-07-14T11:23:35.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.524+0000 7fcc5e19c700 1 -- 172.21.15.40:0/734941377 learned_addr learned my addr 172.21.15.40:0/734941377 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:35.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 -- 172.21.15.40:0/734941377 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 msgr2=0x56073e3ff110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e3ff110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 -- 172.21.15.40:0/734941377 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 msgr2=0x56073e3fc7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.532 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e3fc7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 -- 172.21.15.40:0/734941377 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56073e171de0 con 0x56073e3f24f0 2023-07-14T11:23:35.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fcc500115b0 tx=0x7fcc50009920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=955682cdeff39e46 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc43fff700 1 -- 172.21.15.40:0/734941377 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc5000e040 con 0x56073e3f24f0 2023-07-14T11:23:35.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc43fff700 1 -- 172.21.15.40:0/734941377 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcc5000f040 con 0x56073e3f24f0 2023-07-14T11:23:35.533 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc43fff700 1 -- 172.21.15.40:0/734941377 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc500046f0 con 0x56073e3f24f0 2023-07-14T11:23:35.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.525+0000 7fcc4effd700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 0x7fcc440a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc4effd700 1 -- 172.21.15.40:0/1336855080 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fcc48014070 con 0x7fcc580ff110 2023-07-14T11:23:35.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 -- 172.21.15.40:0/734941377 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 msgr2=0x56073e3f28c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7fcc500115b0 tx=0x7fcc50009920 comp rx=0 tx=0).stop 2023-07-14T11:23:35.534 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 0x7fcc440a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 -- 172.21.15.40:0/734941377 shutdown_connections 2023-07-14T11:23:35.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e3ff110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e3f28c0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 --2- 172.21.15.40:0/734941377 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e3fc7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.535 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 -- 172.21.15.40:0/734941377 >> 172.21.15.40:0/734941377 conn(0x56073e3e7ff0 msgr2=0x56073e3ed930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.536 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.526+0000 7fcc735bab80 1 -- 172.21.15.40:0/734941377 shutdown_connections 2023-07-14T11:23:35.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.527+0000 7fcc735bab80 1 -- 172.21.15.40:0/734941377 wait complete. 2023-07-14T11:23:35.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.527+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 0x7fcc440a3890 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fcc5400b6f0 tx=0x7fcc540058e0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.538 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.527+0000 7fcc735bab80 1 -- start start 2023-07-14T11:23:35.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.527+0000 7fcc735bab80 1 Processor -- start 2023-07-14T11:23:35.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e3fc7c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3fcd30 0x56073e3fa110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.539 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e2e3ac0 con 0x56073e3f24f0 2023-07-14T11:23:35.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e2e37c0 con 0x56073e3f2e00 2023-07-14T11:23:35.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e1c7920 con 0x56073e3fcd30 2023-07-14T11:23:35.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc735bab80 10 client.0 _ll_register_callbacks cb 0x56073e2e1760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:23:35.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.540 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38496/0 (socket says 172.21.15.40:38496) 2023-07-14T11:23:35.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2591886197 learned_addr learned my addr 172.21.15.40:0/2591886197 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:35.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3fcd30 0x56073e3fa110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.528+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e3fc7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2591886197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3fcd30 msgr2=0x56073e3fa110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.541 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3fcd30 0x56073e3fa110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2591886197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 msgr2=0x56073e3fc7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e3fc7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2591886197 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc50011290 con 0x56073e3f2e00 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fcc48009230 tx=0x7fcc48099e30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=bc8b6679d5bfe28 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc417fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:23:35.542 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc48099f30 con 0x56073e3f2e00 2023-07-14T11:23:35.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fcc4809df60 con 0x56073e3f2e00 2023-07-14T11:23:35.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56073e29d4f0 con 0x56073e3f2e00 2023-07-14T11:23:35.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=207) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcc480a2380 con 0x56073e3f2e00 2023-07-14T11:23:35.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc735bab80 10 client.26540 fetch_fsmap learned FSMap version 207 2023-07-14T11:23:35.543 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.529+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=207,monmap=4+}) v3 -- 0x56073e15ac60 con 0x56073e3f2e00 2023-07-14T11:23:35.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.530+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56073e29d4f0 con 0x56073e3f2e00 2023-07-14T11:23:35.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.530+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 207) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fcc4809a200 con 0x56073e3f2e00 2023-07-14T11:23:35.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 10 client.26540 fetch_fsmap finished waiting for FSMap version 207 2023-07-14T11:23:35.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 10 client.26540 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:23:35.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 20 client.26540 populate_metadata read hostname 'smithi040' 2023-07-14T11:23:35.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e40c6d0 0x56073e40eae0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 4 client.26540 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:23:35.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56073e33ceb0 con 0x56073e40c6d0 2023-07-14T11:23:35.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e40c6d0 0x56073e40eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.545 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc417fa700 10 client.26540 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:35.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.531+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e40c6d0 0x56073e40eae0 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=69cdb228d7a1edfa server_cookie=4ebb418b14ac6a89 in_seq=0 out_seq=0 2023-07-14T11:23:35.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.532+0000 7fcc417fa700 1 -- 172.21.15.40:0/2591886197 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x56073e33ceb0 con 0x56073e40c6d0 2023-07-14T11:23:35.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.532+0000 7fcc417fa700 10 client.26540 handle_command_reply: tid=0 2023-07-14T11:23:35.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.532+0000 7fcc735bab80 1 client.26540 shutdown 2023-07-14T11:23:35.546 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 msgr2=0x56073e3f99d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fcc48009230 tx=0x7fcc48099e30 comp rx=0 tx=0).stop 2023-07-14T11:23:35.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 shutdown_connections 2023-07-14T11:23:35.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e40c6d0 0x56073e40eae0 crc :-1 s=READY pgs=115 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3fcd30 0x56073e3fa110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e3fc7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2591886197 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3f2e00 0x56073e3f99d0 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 >> 172.21.15.40:0/2591886197 conn(0x56073e3e7ff0 msgr2=0x56073e3ed520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc417fa700 0 client.26540 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:35.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 shutdown_connections 2023-07-14T11:23:35.548 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 1 -- 172.21.15.40:0/2591886197 wait complete. 2023-07-14T11:23:35.549 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.533+0000 7fcc735bab80 20 client.26540 trim_cache size 0 max 16384 2023-07-14T11:23:35.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 Processor -- start 2023-07-14T11:23:35.559 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 -- start start 2023-07-14T11:23:35.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e403810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e408980 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e33ceb0 con 0x56073e3f24f0 2023-07-14T11:23:35.561 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e29d4f0 con 0x56073e3fcd30 2023-07-14T11:23:35.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.556+0000 7fcc735bab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e409e80 con 0x56073e3f2e00 2023-07-14T11:23:35.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58528/0 (socket says 172.21.15.40:58528) 2023-07-14T11:23:35.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2433212239 learned_addr learned my addr 172.21.15.40:0/2433212239 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:35.562 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e403810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e408980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2433212239 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 msgr2=0x56073e408980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e408980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2433212239 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 msgr2=0x56073e403810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.563 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e403810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 -- 172.21.15.40:0/2433212239 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56073e198250 con 0x56073e3f2e00 2023-07-14T11:23:35.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e403810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:23:35.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.557+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fcc48005950 tx=0x7fcc48000bc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=9279ab2e99c9bc9 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.558+0000 7fcc4cbff700 1 -- 172.21.15.40:0/2433212239 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc480a2380 con 0x56073e3f2e00 2023-07-14T11:23:35.564 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.558+0000 7fcc4cbff700 1 -- 172.21.15.40:0/2433212239 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcc48099890 con 0x56073e3f2e00 2023-07-14T11:23:35.565 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.558+0000 7fcc4cbff700 1 -- 172.21.15.40:0/2433212239 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc480a2110 con 0x56073e3f2e00 2023-07-14T11:23:35.567 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- 172.21.15.40:0/2433212239 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 msgr2=0x56073e407fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7fcc48005950 tx=0x7fcc48000bc0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- 172.21.15.40:0/2433212239 shutdown_connections 2023-07-14T11:23:35.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e408980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.568 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f24f0 0x56073e403810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 --2- 172.21.15.40:0/2433212239 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f2e00 0x56073e407fb0 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- 172.21.15.40:0/2433212239 >> 172.21.15.40:0/2433212239 conn(0x56073e3e7ff0 msgr2=0x56073e3ec060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- 172.21.15.40:0/2433212239 shutdown_connections 2023-07-14T11:23:35.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- 172.21.15.40:0/2433212239 wait complete. 2023-07-14T11:23:35.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 -- start start 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.559+0000 7fcc735bab80 1 Processor -- start 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e4023c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e402900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e4606f0 con 0x56073e3f2e00 2023-07-14T11:23:35.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e460830 con 0x56073e3fcd30 2023-07-14T11:23:35.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56073e4609f0 con 0x56073e3f24f0 2023-07-14T11:23:35.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc735bab80 10 client.0 _ll_register_callbacks cb 0x56073e2e1760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:23:35.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc5d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc5d99b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:58540/0 (socket says 172.21.15.40:58540) 2023-07-14T11:23:35.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc5d99b700 1 -- 172.21.15.40:0/1890082344 learned_addr learned my addr 172.21.15.40:0/1890082344 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:23:35.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc5e19c700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e402900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.560+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e4023c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 -- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 msgr2=0x56073e402900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e402900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.572 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 -- 172.21.15.40:0/1890082344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 msgr2=0x56073e4023c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:23:35.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e4023c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 -- 172.21.15.40:0/1890082344 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56073e15ac60 con 0x56073e3f24f0 2023-07-14T11:23:35.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e4023c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:23:35.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc5d99b700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fcc54009990 tx=0x7fcc54005fd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=a98790d1604f147 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:23:35.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc417fa700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:23:35.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fcc54032bd0 con 0x56073e3f24f0 2023-07-14T11:23:35.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fcc540325a0 con 0x56073e3f24f0 2023-07-14T11:23:35.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.561+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fcc5000f040 con 0x56073e3f24f0 2023-07-14T11:23:35.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=207) v2 ==== 24+0+0 (secure 0 0 0) 0x7fcc54032d30 con 0x56073e3f24f0 2023-07-14T11:23:35.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 10 client.26509 fetch_fsmap learned FSMap version 207 2023-07-14T11:23:35.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=207,monmap=4+}) v3 -- 0x56073e1702b0 con 0x56073e3f24f0 2023-07-14T11:23:35.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fcc5000f040 con 0x56073e3f24f0 2023-07-14T11:23:35.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 207) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fcc54038020 con 0x56073e3f24f0 2023-07-14T11:23:35.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 10 client.26509 fetch_fsmap finished waiting for FSMap version 207 2023-07-14T11:23:35.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 10 client.26509 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:23:35.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 20 client.26509 populate_metadata read hostname 'smithi040' 2023-07-14T11:23:35.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e462130 0x56073e464540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:23:35.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 4 client.26509 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:23:35.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.562+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56073e2ff050 con 0x56073e462130 2023-07-14T11:23:35.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e462130 0x56073e464540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:23:35.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc5d19a700 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e462130 0x56073e464540 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=8bbe1325a8c01439 server_cookie=c40db7b1b0fa4766 in_seq=0 out_seq=0 2023-07-14T11:23:35.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc417fa700 10 client.26509 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:35.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc417fa700 1 -- 172.21.15.40:0/1890082344 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x56073e2ff050 con 0x56073e462130 2023-07-14T11:23:35.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc417fa700 10 client.26509 handle_command_reply: tid=0 2023-07-14T11:23:35.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc735bab80 1 client.26509 shutdown 2023-07-14T11:23:35.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 msgr2=0x56073e401e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.563+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7fcc54009990 tx=0x7fcc54005fd0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 shutdown_connections 2023-07-14T11:23:35.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56073e462130 0x56073e464540 crc :-1 s=READY pgs=116 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.578 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56073e3fcd30 0x56073e402900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc417fa700 0 client.26509 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56073e3f24f0 0x56073e401e80 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 --2- 172.21.15.40:0/1890082344 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56073e3f2e00 0x56073e4023c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 >> 172.21.15.40:0/1890082344 conn(0x56073e3e7ff0 msgr2=0x56073e408980 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 shutdown_connections 2023-07-14T11:23:35.579 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 1 -- 172.21.15.40:0/1890082344 wait complete. 2023-07-14T11:23:35.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.564+0000 7fcc735bab80 20 client.26509 trim_cache size 0 max 16384 2023-07-14T11:23:35.580 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:23:35.580 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:23:35.581 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:23:35.581 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:23:35.581 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:23:35.581 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:23:35.581 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:23:35.582 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:23:35.582 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:23:35.582 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:23:35.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 msgr2=0x7fcc440a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 0x7fcc440a3890 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fcc5400b6f0 tx=0x7fcc540058e0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 msgr2=0x7fcc5819dec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:23:35.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7fcc4800e970 tx=0x7fcc4800ec80 comp rx=0 tx=0).stop 2023-07-14T11:23:35.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 shutdown_connections 2023-07-14T11:23:35.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fcc440a13e0 0x7fcc440a3890 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcc5810c3b0 0x7fcc581979f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fcc580fe800 0x7fcc5819d980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 --2- 172.21.15.40:0/1336855080 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fcc580ff110 0x7fcc5819dec0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:23:35.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 >> 172.21.15.40:0/1336855080 conn(0x7fcc580745d0 msgr2=0x7fcc580fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:23:35.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 shutdown_connections 2023-07-14T11:23:35.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:23:35.565+0000 7fcc5e99d700 1 -- 172.21.15.40:0/1336855080 wait complete. 2023-07-14T11:23:35.587 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:23:35.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: 897 slow requests, 5 included below; oldest blocked for > 576.451256 secs 2023-07-14T11:23:36.141 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: slow request 574.350120 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.142 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: slow request 574.321406 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.142 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: slow request 575.062209 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.142 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: slow request 575.036304 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.142 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: slow request 575.007881 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.143 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:35 smithi040 ceph-mon[97954]: pgmap v356: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:36.143 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: 897 slow requests, 5 included below; oldest blocked for > 576.451256 secs 2023-07-14T11:23:36.144 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: slow request 574.350120 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.144 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: slow request 574.321406 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.144 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: slow request 575.062209 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.144 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: slow request 575.036304 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.144 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: slow request 575.007881 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.145 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:35 smithi045 ceph-mon[128791]: pgmap v356: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:36.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: 897 slow requests, 5 included below; oldest blocked for > 576.451256 secs 2023-07-14T11:23:36.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: slow request 574.350120 seconds old, received at 2023-07-14T11:14:00.203716+0000: client_request(mds.1:2839 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: slow request 574.321406 seconds old, received at 2023-07-14T11:14:00.232430+0000: client_request(mds.1:2871 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: slow request 575.062209 seconds old, received at 2023-07-14T11:13:59.491627+0000: client_request(mds.1:2008 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: slow request 575.036304 seconds old, received at 2023-07-14T11:13:59.517532+0000: client_request(mds.1:2040 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: slow request 575.007881 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:36.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:35 smithi130 ceph-mon[129405]: pgmap v356: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:37 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:38.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:37 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:38.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:37 smithi130 ceph-mon[129405]: pgmap v357: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:37 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:37 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:37 smithi040 ceph-mon[97954]: pgmap v357: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:37 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:37 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:38.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:37 smithi045 ceph-mon[128791]: pgmap v357: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: 881 slow requests, 5 included below; oldest blocked for > 577.167137 secs 2023-07-14T11:23:39.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: slow request 576.810820 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: slow request 576.805107 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: slow request 576.799357 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: slow request 576.793710 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:39.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:38 smithi130 ceph-mon[129405]: slow request 576.787990 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: 881 slow requests, 5 included below; oldest blocked for > 577.167137 secs 2023-07-14T11:23:39.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: slow request 576.810820 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: slow request 576.805107 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: slow request 576.799357 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: slow request 576.793710 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:39.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:38 smithi040 ceph-mon[97954]: slow request 576.787990 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: 881 slow requests, 5 included below; oldest blocked for > 577.167137 secs 2023-07-14T11:23:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: slow request 576.810820 seconds old, received at 2023-07-14T11:14:01.289873+0000: peer_request:mds.1:2057 currently dispatched 2023-07-14T11:23:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: slow request 576.805107 seconds old, received at 2023-07-14T11:14:01.295586+0000: peer_request:mds.1:2089 currently dispatched 2023-07-14T11:23:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: slow request 576.799357 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: slow request 576.793710 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:38 smithi045 ceph-mon[128791]: slow request 576.787990 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:40.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:40.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:39 smithi130 ceph-mon[129405]: pgmap v358: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:40.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:39 smithi040 ceph-mon[97954]: pgmap v358: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:39 smithi045 ceph-mon[128791]: pgmap v358: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:41.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: 875 slow requests, 5 included below; oldest blocked for > 581.521252 secs 2023-07-14T11:23:41.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: slow request 580.077876 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: slow request 580.051157 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: slow request 580.023331 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: slow request 579.992348 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:40 smithi130 ceph-mon[129405]: slow request 579.966690 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: 875 slow requests, 5 included below; oldest blocked for > 581.521252 secs 2023-07-14T11:23:41.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: slow request 580.077876 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: slow request 580.051157 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: slow request 580.023331 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: slow request 579.992348 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:40 smithi040 ceph-mon[97954]: slow request 579.966690 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: 875 slow requests, 5 included below; oldest blocked for > 581.521252 secs 2023-07-14T11:23:41.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: slow request 580.077876 seconds old, received at 2023-07-14T11:13:59.545954+0000: client_request(mds.1:2072 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: slow request 580.051157 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: slow request 580.023331 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: slow request 579.992348 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:41.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:40 smithi045 ceph-mon[128791]: slow request 579.966690 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:41 smithi130 ceph-mon[129405]: pgmap v359: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:41 smithi040 ceph-mon[97954]: pgmap v359: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:41 smithi045 ceph-mon[128791]: pgmap v359: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: pgmap v360: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: 859 slow requests, 5 included below; oldest blocked for > 582.167235 secs 2023-07-14T11:23:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: slow request 581.799456 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: slow request 581.793809 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: slow request 581.788088 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: slow request 581.782384 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:43 smithi130 ceph-mon[129405]: slow request 581.776598 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: pgmap v360: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: 859 slow requests, 5 included below; oldest blocked for > 582.167235 secs 2023-07-14T11:23:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: slow request 581.799456 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: slow request 581.793809 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: slow request 581.788088 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: slow request 581.782384 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:44.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:43 smithi040 ceph-mon[97954]: slow request 581.776598 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: pgmap v360: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: 859 slow requests, 5 included below; oldest blocked for > 582.167235 secs 2023-07-14T11:23:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: slow request 581.799456 seconds old, received at 2023-07-14T11:14:01.301336+0000: peer_request:mds.1:2121 currently dispatched 2023-07-14T11:23:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: slow request 581.793809 seconds old, received at 2023-07-14T11:14:01.306983+0000: peer_request:mds.1:2153 currently dispatched 2023-07-14T11:23:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: slow request 581.788088 seconds old, received at 2023-07-14T11:14:01.312703+0000: peer_request:mds.1:2185 currently dispatched 2023-07-14T11:23:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: slow request 581.782384 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:43 smithi045 ceph-mon[128791]: slow request 581.776598 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: pgmap v361: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: 851 slow requests, 5 included below; oldest blocked for > 586.661506 secs 2023-07-14T11:23:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: slow request 585.191411 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: slow request 585.163586 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: slow request 585.132602 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: slow request 585.106944 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:45 smithi130 ceph-mon[129405]: slow request 585.081681 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: pgmap v361: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: 851 slow requests, 5 included below; oldest blocked for > 586.661506 secs 2023-07-14T11:23:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: slow request 585.191411 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: slow request 585.163586 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: slow request 585.132602 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: slow request 585.106944 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:45 smithi040 ceph-mon[97954]: slow request 585.081681 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: pgmap v361: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: 851 slow requests, 5 included below; oldest blocked for > 586.661506 secs 2023-07-14T11:23:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: slow request 585.191411 seconds old, received at 2023-07-14T11:13:59.572674+0000: client_request(mds.1:2104 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: slow request 585.163586 seconds old, received at 2023-07-14T11:13:59.600500+0000: client_request(mds.1:2136 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: slow request 585.132602 seconds old, received at 2023-07-14T11:13:59.631483+0000: client_request(mds.1:2168 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: slow request 585.106944 seconds old, received at 2023-07-14T11:13:59.657141+0000: client_request(mds.1:2200 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:45 smithi045 ceph-mon[128791]: slow request 585.081681 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:48.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:47 smithi130 ceph-mon[129405]: pgmap v362: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:47 smithi040 ceph-mon[97954]: pgmap v362: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:47 smithi045 ceph-mon[128791]: pgmap v362: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: 835 slow requests, 5 included below; oldest blocked for > 587.167371 secs 2023-07-14T11:23:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: slow request 586.782519 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: slow request 586.776733 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: slow request 586.770660 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: slow request 586.764667 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:48 smithi130 ceph-mon[129405]: slow request 586.758841 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: 835 slow requests, 5 included below; oldest blocked for > 587.167371 secs 2023-07-14T11:23:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: slow request 586.782519 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: slow request 586.776733 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: slow request 586.770660 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: slow request 586.764667 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:48 smithi040 ceph-mon[97954]: slow request 586.758841 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: 835 slow requests, 5 included below; oldest blocked for > 587.167371 secs 2023-07-14T11:23:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: slow request 586.782519 seconds old, received at 2023-07-14T11:14:01.318407+0000: peer_request:mds.1:2217 currently dispatched 2023-07-14T11:23:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: slow request 586.776733 seconds old, received at 2023-07-14T11:14:01.324194+0000: peer_request:mds.1:2249 currently dispatched 2023-07-14T11:23:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: slow request 586.770660 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: slow request 586.764667 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:48 smithi045 ceph-mon[128791]: slow request 586.758841 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:49 smithi130 ceph-mon[129405]: pgmap v363: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:49 smithi040 ceph-mon[97954]: pgmap v363: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:49 smithi045 ceph-mon[128791]: pgmap v363: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 9 op/s 2023-07-14T11:23:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: 829 slow requests, 5 included below; oldest blocked for > 591.682727 secs 2023-07-14T11:23:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: slow request 590.102902 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: slow request 590.077848 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: slow request 590.048422 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: slow request 590.008174 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:50 smithi130 ceph-mon[129405]: slow request 589.979728 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: 829 slow requests, 5 included below; oldest blocked for > 591.682727 secs 2023-07-14T11:23:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: slow request 590.102902 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: slow request 590.077848 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: slow request 590.048422 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: slow request 590.008174 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:50 smithi040 ceph-mon[97954]: slow request 589.979728 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: 829 slow requests, 5 included below; oldest blocked for > 591.682727 secs 2023-07-14T11:23:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: slow request 590.102902 seconds old, received at 2023-07-14T11:13:59.682404+0000: client_request(mds.1:2232 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: slow request 590.077848 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: slow request 590.048422 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: slow request 590.008174 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:50 smithi045 ceph-mon[128791]: slow request 589.979728 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:51 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:52.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:51 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:52.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:51 smithi130 ceph-mon[129405]: pgmap v364: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:51 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:51 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:51 smithi040 ceph-mon[97954]: pgmap v364: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:51 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:23:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:51 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:52.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:51 smithi045 ceph-mon[128791]: pgmap v364: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: pgmap v365: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:23:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: 813 slow requests, 5 included below; oldest blocked for > 592.167512 secs 2023-07-14T11:23:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: slow request 591.770801 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: slow request 591.764809 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: slow request 591.758982 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: slow request 591.753017 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:53 smithi130 ceph-mon[129405]: slow request 591.747037 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: pgmap v365: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:23:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: 813 slow requests, 5 included below; oldest blocked for > 592.167512 secs 2023-07-14T11:23:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: slow request 591.770801 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: slow request 591.764809 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: slow request 591.758982 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: slow request 591.753017 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:53 smithi040 ceph-mon[97954]: slow request 591.747037 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: pgmap v365: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:23:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: 813 slow requests, 5 included below; oldest blocked for > 592.167512 secs 2023-07-14T11:23:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: slow request 591.770801 seconds old, received at 2023-07-14T11:14:01.330267+0000: peer_request:mds.1:2281 currently dispatched 2023-07-14T11:23:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: slow request 591.764809 seconds old, received at 2023-07-14T11:14:01.336259+0000: peer_request:mds.1:2313 currently dispatched 2023-07-14T11:23:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: slow request 591.758982 seconds old, received at 2023-07-14T11:14:01.342086+0000: peer_request:mds.1:2345 currently dispatched 2023-07-14T11:23:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: slow request 591.753017 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:53 smithi045 ceph-mon[128791]: slow request 591.747037 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: pgmap v366: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: 807 slow requests, 5 included below; oldest blocked for > 596.721291 secs 2023-07-14T11:23:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: slow request 595.116413 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: slow request 595.086987 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: slow request 595.046739 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: slow request 595.018293 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: slow request 594.991722 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:55 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:23:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: pgmap v366: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: 807 slow requests, 5 included below; oldest blocked for > 596.721291 secs 2023-07-14T11:23:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: slow request 595.116413 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: slow request 595.086987 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: slow request 595.046739 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: slow request 595.018293 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: slow request 594.991722 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:55 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:23:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: pgmap v366: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:23:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: 807 slow requests, 5 included below; oldest blocked for > 596.721291 secs 2023-07-14T11:23:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: slow request 595.116413 seconds old, received at 2023-07-14T11:13:59.707458+0000: client_request(mds.1:2264 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: slow request 595.086987 seconds old, received at 2023-07-14T11:13:59.736884+0000: client_request(mds.1:2296 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: slow request 595.046739 seconds old, received at 2023-07-14T11:13:59.777132+0000: client_request(mds.1:2328 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: slow request 595.018293 seconds old, received at 2023-07-14T11:13:59.805578+0000: client_request(mds.1:2360 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: slow request 594.991722 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:23:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:55 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:23:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:57 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:57 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:57 smithi130 ceph-mon[129405]: pgmap v367: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:58.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:57 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:57 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:57 smithi040 ceph-mon[97954]: pgmap v367: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:23:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:23:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:57 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:57 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:58.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:57 smithi045 ceph-mon[128791]: pgmap v367: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:23:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:23:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:23:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: 791 slow requests, 5 included below; oldest blocked for > 597.167638 secs 2023-07-14T11:23:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: slow request 596.753144 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: slow request 596.747163 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: slow request 596.741285 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:23:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: slow request 596.735407 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:23:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:58 smithi130 ceph-mon[129405]: slow request 596.729590 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:23:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:23:59.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:23:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: 791 slow requests, 5 included below; oldest blocked for > 597.167638 secs 2023-07-14T11:23:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: slow request 596.753144 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: slow request 596.747163 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: slow request 596.741285 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:23:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: slow request 596.735407 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:23:59.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:58 smithi040 ceph-mon[97954]: slow request 596.729590 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:23:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:23:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:23:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:23:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: 791 slow requests, 5 included below; oldest blocked for > 597.167638 secs 2023-07-14T11:23:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: slow request 596.753144 seconds old, received at 2023-07-14T11:14:01.348051+0000: peer_request:mds.1:2377 currently dispatched 2023-07-14T11:23:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: slow request 596.747163 seconds old, received at 2023-07-14T11:14:01.354031+0000: peer_request:mds.1:2409 currently dispatched 2023-07-14T11:23:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: slow request 596.741285 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:23:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: slow request 596.735407 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:23:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:58 smithi045 ceph-mon[128791]: slow request 596.729590 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:24:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:59 smithi130 ceph-mon[129405]: pgmap v368: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:23:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:59 smithi040 ceph-mon[97954]: pgmap v368: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:23:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:59 smithi045 ceph-mon[128791]: pgmap v368: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:23:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: 783 slow requests, 5 included below; oldest blocked for > 601.802137 secs 2023-07-14T11:24:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: slow request 600.072567 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: slow request 600.046505 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: slow request 600.020531 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: slow request 599.994407 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: slow request 599.967365 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:01.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:00 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: 783 slow requests, 5 included below; oldest blocked for > 601.802137 secs 2023-07-14T11:24:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: slow request 600.072567 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: slow request 600.046505 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: slow request 600.020531 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: slow request 599.994407 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: slow request 599.967365 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:00 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: 783 slow requests, 5 included below; oldest blocked for > 601.802137 secs 2023-07-14T11:24:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: slow request 600.072567 seconds old, received at 2023-07-14T11:13:59.832149+0000: client_request(mds.1:2392 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: slow request 600.046505 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: slow request 600.020531 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: slow request 599.994407 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: slow request 599.967365 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:00 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:01 smithi130 ceph-mon[129405]: pgmap v369: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:24:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:01 smithi040 ceph-mon[97954]: pgmap v369: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:24:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:01 smithi045 ceph-mon[128791]: pgmap v369: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 13 op/s 2023-07-14T11:24:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: pgmap v370: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: 770 slow requests, 5 included below; oldest blocked for > 602.167732 secs 2023-07-14T11:24:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: slow request 601.741378 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:24:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: slow request 601.735501 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:24:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: slow request 601.729683 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:24:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: slow request 601.723768 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:03 smithi130 ceph-mon[129405]: slow request 601.717824 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: pgmap v370: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: 770 slow requests, 5 included below; oldest blocked for > 602.167732 secs 2023-07-14T11:24:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: slow request 601.741378 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:24:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: slow request 601.735501 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:24:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: slow request 601.729683 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:24:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: slow request 601.723768 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:03 smithi040 ceph-mon[97954]: slow request 601.717824 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: pgmap v370: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 9 op/s 2023-07-14T11:24:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: 770 slow requests, 5 included below; oldest blocked for > 602.167732 secs 2023-07-14T11:24:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: slow request 601.741378 seconds old, received at 2023-07-14T11:14:01.359909+0000: peer_request:mds.1:2441 currently dispatched 2023-07-14T11:24:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: slow request 601.735501 seconds old, received at 2023-07-14T11:14:01.365787+0000: peer_request:mds.1:2473 currently dispatched 2023-07-14T11:24:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: slow request 601.729683 seconds old, received at 2023-07-14T11:14:01.371604+0000: peer_request:mds.1:2505 currently dispatched 2023-07-14T11:24:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: slow request 601.723768 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:03 smithi045 ceph-mon[128791]: slow request 601.717824 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:04 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:04 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:04 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:04 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:04 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:04 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:05.588 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:24:05.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/774670736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4106ea0 msgr2=0x7f1cc410ea70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/774670736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4106ea0 0x7f1cc410ea70 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7f1cbc00b0a0 tx=0x7f1cbc00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/774670736 shutdown_connections 2023-07-14T11:24:05.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/774670736 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4106ea0 0x7f1cc410ea70 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/774670736 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4101e60 0x7f1cc4102230 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/774670736 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 0x7f1cc41067c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/774670736 >> 172.21.15.40:0/774670736 conn(0x7f1cc40fd690 msgr2=0x7f1cc40ffaa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/774670736 shutdown_connections 2023-07-14T11:24:05.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.821+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/774670736 wait complete. 2023-07-14T11:24:05.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.823+0000 7f1cc9f9d700 1 Processor -- start 2023-07-14T11:24:05.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.823+0000 7f1cc9f9d700 1 -- start start 2023-07-14T11:24:05.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 0x7f1cc419df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4106ea0 0x7f1cc4197aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cc4198110 con 0x7f1cc4102800 2023-07-14T11:24:05.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cc4198280 con 0x7f1cc4101e60 2023-07-14T11:24:05.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc9f9d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1cc4198440 con 0x7f1cc4106ea0 2023-07-14T11:24:05.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56686/0 (socket says 172.21.15.40:56686) 2023-07-14T11:24:05.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/627745808 learned_addr learned my addr 172.21.15.40:0/627745808 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:05.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 0x7f1cc419df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc979c700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4106ea0 0x7f1cc4197aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/627745808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4106ea0 msgr2=0x7f1cc4197aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4106ea0 0x7f1cc4197aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 msgr2=0x7f1cc419df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 0x7f1cc419df70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.824+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/627745808 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1cb8009520 con 0x7f1cc4101e60 2023-07-14T11:24:05.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f1cb800eb30 tx=0x7f1cb800ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc1ffb700 1 -- 172.21.15.40:0/627745808 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb8009880 con 0x7f1cc4101e60 2023-07-14T11:24:05.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc1ffb700 1 -- 172.21.15.40:0/627745808 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1cb80044e0 con 0x7f1cc4101e60 2023-07-14T11:24:05.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1cbc009d00 con 0x7f1cc4101e60 2023-07-14T11:24:05.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc1ffb700 1 -- 172.21.15.40:0/627745808 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb8003d80 con 0x7f1cc4101e60 2023-07-14T11:24:05.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.825+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cc4198910 con 0x7f1cc4101e60 2023-07-14T11:24:05.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.829+0000 7f1cc1ffb700 1 -- 172.21.15.40:0/627745808 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f1cb8003f20 con 0x7f1cc4101e60 2023-07-14T11:24:05.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.830+0000 7f1cc1ffb700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 0x7f1cb40a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.831+0000 7f1cc1ffb700 1 -- 172.21.15.40:0/627745808 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f1cb8014070 con 0x7f1cc4101e60 2023-07-14T11:24:05.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.831+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 0x7f1cb40a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.831+0000 7f1cdeb1eb80 1 Processor -- start 2023-07-14T11:24:05.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 0x7f1cb40a3890 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4100090 tx=0x7f1cb0009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 -- start start 2023-07-14T11:24:05.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 0x55c2f6f2b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2e110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.836 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6e26cb0 con 0x55c2f6f21e00 2023-07-14T11:24:05.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6e79a40 con 0x55c2f6f2bd30 2023-07-14T11:24:05.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6e72ed0 con 0x55c2f6f214f0 2023-07-14T11:24:05.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.837 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 0x55c2f6f2b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:46456/0 (socket says 172.21.15.40:46456) 2023-07-14T11:24:05.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/3820933921 learned_addr learned my addr 172.21.15.40:0/3820933921 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:05.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.832+0000 7f1cc979c700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2e110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/3820933921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 msgr2=0x55c2f6f2e110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2e110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/3820933921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 msgr2=0x55c2f6f2b7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 0x55c2f6f2b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/3820933921 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c2f6ca0de0 con 0x55c2f6f214f0 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 0x55c2f6f2b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc979c700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:24:05.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f1cb80a2040 tx=0x7f1cb8099b00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=be2bcd5962c0f8a1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1caeffd700 1 -- 172.21.15.40:0/3820933921 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb809a120 con 0x55c2f6f214f0 2023-07-14T11:24:05.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1caeffd700 1 -- 172.21.15.40:0/3820933921 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1cb809e150 con 0x55c2f6f214f0 2023-07-14T11:24:05.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.833+0000 7f1caeffd700 1 -- 172.21.15.40:0/3820933921 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb809a5e0 con 0x55c2f6f214f0 2023-07-14T11:24:05.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3820933921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 msgr2=0x55c2f6f218c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f1cb80a2040 tx=0x7f1cb8099b00 comp rx=0 tx=0).stop 2023-07-14T11:24:05.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3820933921 shutdown_connections 2023-07-14T11:24:05.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2e110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f218c0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3820933921 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f21e00 0x55c2f6f2b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3820933921 >> 172.21.15.40:0/3820933921 conn(0x55c2f6f16ff0 msgr2=0x55c2f6f1c930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3820933921 shutdown_connections 2023-07-14T11:24:05.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.834+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3820933921 wait complete. 2023-07-14T11:24:05.842 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 -- start start 2023-07-14T11:24:05.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 Processor -- start 2023-07-14T11:24:05.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f2b7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f21e00 0x55c2f6f28a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.845 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6e12ac0 con 0x55c2f6f214f0 2023-07-14T11:24:05.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6e127c0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6cf6920 con 0x55c2f6f21e00 2023-07-14T11:24:05.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.835+0000 7f1cdeb1eb80 10 client.0 _ll_register_callbacks cb 0x55c2f6e10760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:24:05.846 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f2b7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56716/0 (socket says 172.21.15.40:56716) 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 -- 172.21.15.40:0/2997613960 learned_addr learned my addr 172.21.15.40:0/2997613960 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f21e00 0x55c2f6f28a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 -- 172.21.15.40:0/2997613960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f21e00 msgr2=0x55c2f6f28a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.847 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f21e00 0x55c2f6f28a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 -- 172.21.15.40:0/2997613960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 msgr2=0x55c2f6f2b7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f2b7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc979c700 1 -- 172.21.15.40:0/2997613960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1cbc009d00 con 0x55c2f6f2bd30 2023-07-14T11:24:05.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.836+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f2b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:24:05.848 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1cc979c700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f1cbc00b0a0 tx=0x7f1cbc00bf90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=67b62e8334782d7e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1ca3fff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:24:05.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cbc00e040 con 0x55c2f6f2bd30 2023-07-14T11:24:05.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c2f6dcc4f0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f1cbc0074e0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.849 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=220) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1cbc0076a0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1cdeb1eb80 10 client.26558 fetch_fsmap learned FSMap version 220 2023-07-14T11:24:05.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=220,monmap=4+}) v3 -- 0x55c2f6c89c60 con 0x55c2f6f2bd30 2023-07-14T11:24:05.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.837+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c2f6dcc4f0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 220) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1cbc023030 con 0x55c2f6f2bd30 2023-07-14T11:24:05.850 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 10 client.26558 fetch_fsmap finished waiting for FSMap version 220 2023-07-14T11:24:05.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 10 client.26558 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:24:05.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 20 client.26558 populate_metadata read hostname 'smithi040' 2023-07-14T11:24:05.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f3b720 0x55c2f6f3db30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 4 client.26558 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:24:05.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c2f6e6beb0 con 0x55c2f6f3b720 2023-07-14T11:24:05.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.838+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f3b720 0x55c2f6f3db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.839+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f3b720 0x55c2f6f3db30 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d2eed40baf8e96ad server_cookie=3ea875d650ff9713 in_seq=0 out_seq=0 2023-07-14T11:24:05.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.839+0000 7f1ca3fff700 10 client.26558 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:05.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.839+0000 7f1ca3fff700 1 -- 172.21.15.40:0/2997613960 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c2f6e6beb0 con 0x55c2f6f3b720 2023-07-14T11:24:05.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.839+0000 7f1ca3fff700 10 client.26558 handle_command_reply: tid=0 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.839+0000 7f1cdeb1eb80 1 client.26558 shutdown 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 msgr2=0x55c2f6f29140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f1cbc00b0a0 tx=0x7f1cbc00bf90 comp rx=0 tx=0).stop 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 shutdown_connections 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f3b720 0x55c2f6f3db30 crc :-1 s=READY pgs=117 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f29140 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1ca3fff700 0 client.26558 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:05.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f2b7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/2997613960 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f21e00 0x55c2f6f28a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 >> 172.21.15.40:0/2997613960 conn(0x55c2f6f16ff0 msgr2=0x55c2f6f1c520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 shutdown_connections 2023-07-14T11:24:05.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/2997613960 wait complete. 2023-07-14T11:24:05.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.840+0000 7f1cdeb1eb80 20 client.26558 trim_cache size 0 max 16384 2023-07-14T11:24:05.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.876+0000 7f1cdeb1eb80 1 Processor -- start 2023-07-14T11:24:05.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.876+0000 7f1cdeb1eb80 1 -- start start 2023-07-14T11:24:05.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32810 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f1bb00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2b320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f32d90 con 0x55c2f6f214f0 2023-07-14T11:24:05.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f3db00 con 0x55c2f6f21e00 2023-07-14T11:24:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f1c040 con 0x55c2f6f2bd30 2023-07-14T11:24:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f1bb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc3fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f1bb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:56728/0 (socket says 172.21.15.40:56728) 2023-07-14T11:24:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc3fff700 1 -- 172.21.15.40:0/3710958285 learned_addr learned my addr 172.21.15.40:0/3710958285 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:05.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2b320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 -- 172.21.15.40:0/3710958285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 msgr2=0x55c2f6f1bb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f1bb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 -- 172.21.15.40:0/3710958285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 msgr2=0x55c2f6f32810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.883 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.877+0000 7f1cc979c700 1 -- 172.21.15.40:0/3710958285 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c2f6cc7250 con 0x55c2f6f2bd30 2023-07-14T11:24:05.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.878+0000 7f1cc979c700 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2b320 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f1cbc0062a0 tx=0x7f1cbc009780 comp rx=0 tx=0).ready entity=mon.1 client_cookie=8b38553a91e7ecc2 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.878+0000 7f1caf7fe700 1 -- 172.21.15.40:0/3710958285 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cbc00e070 con 0x55c2f6f2bd30 2023-07-14T11:24:05.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.878+0000 7f1caf7fe700 1 -- 172.21.15.40:0/3710958285 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f1cbc009b80 con 0x55c2f6f2bd30 2023-07-14T11:24:05.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.878+0000 7f1caf7fe700 1 -- 172.21.15.40:0/3710958285 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cbc01dbc0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3710958285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 msgr2=0x55c2f6f2b320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2b320 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f1cbc0062a0 tx=0x7f1cbc009780 comp rx=0 tx=0).stop 2023-07-14T11:24:05.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3710958285 shutdown_connections 2023-07-14T11:24:05.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f2b320 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/3710958285 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f1bb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3710958285 >> 172.21.15.40:0/3710958285 conn(0x55c2f6f16ff0 msgr2=0x55c2f6f1b060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3710958285 shutdown_connections 2023-07-14T11:24:05.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/3710958285 wait complete. 2023-07-14T11:24:05.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.879+0000 7f1cdeb1eb80 1 -- start start 2023-07-14T11:24:05.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 Processor -- start 2023-07-14T11:24:05.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f30930 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f30e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f313f0 con 0x55c2f6f2bd30 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f31530 con 0x55c2f6f21e00 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c2f6f316f0 con 0x55c2f6f214f0 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cdeb1eb80 10 client.0 _ll_register_callbacks cb 0x7f1cbc009f20 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.880+0000 7f1cc8f9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:46472/0 (socket says 172.21.15.40:46472) 2023-07-14T11:24:05.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/113708450 learned_addr learned my addr 172.21.15.40:0/113708450 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:05.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc979c700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f30e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 msgr2=0x55c2f6f30930 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:24:05.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f30930 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/113708450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 msgr2=0x55c2f6f30e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:05.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f30e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 -- 172.21.15.40:0/113708450 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c2f6c89c60 con 0x55c2f6f214f0 2023-07-14T11:24:05.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc979c700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f30e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:24:05.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1cc8f9b700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1cb800eb00 tx=0x7f1cb801e420 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f3cac9b3bdd9363d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:05.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1ca3fff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:24:05.891 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1cb8096830 con 0x55c2f6f214f0 2023-07-14T11:24:05.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.881+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(103..103 src has 1..103) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f1cb8096990 con 0x55c2f6f214f0 2023-07-14T11:24:05.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c2f6e2e050 con 0x55c2f6f214f0 2023-07-14T11:24:05.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=220) v2 ==== 24+0+0 (secure 0 0 0) 0x7f1cb80b82b0 con 0x55c2f6f214f0 2023-07-14T11:24:05.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 10 client.26527 fetch_fsmap learned FSMap version 220 2023-07-14T11:24:05.892 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=220,monmap=4+}) v3 -- 0x55c2f6c9f2b0 con 0x55c2f6f214f0 2023-07-14T11:24:05.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c2f6e2e050 con 0x55c2f6f214f0 2023-07-14T11:24:05.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 220) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f1cb809a380 con 0x55c2f6f214f0 2023-07-14T11:24:05.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 10 client.26527 fetch_fsmap finished waiting for FSMap version 220 2023-07-14T11:24:05.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 10 client.26527 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:24:05.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 20 client.26527 populate_metadata read hostname 'smithi040' 2023-07-14T11:24:05.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f90750 0x55c2f6f92b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:05.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 4 client.26527 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:24:05.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.882+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c2f6cc1f70 con 0x55c2f6f90750 2023-07-14T11:24:05.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f90750 0x55c2f6f92b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:05.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1cc3fff700 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f90750 0x55c2f6f92b60 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=261a8dd39a40e405 server_cookie=c85d6f7c3b120bdb in_seq=0 out_seq=0 2023-07-14T11:24:05.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1ca3fff700 10 client.26527 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:05.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1ca3fff700 1 -- 172.21.15.40:0/113708450 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55c2f6cc1f70 con 0x55c2f6f90750 2023-07-14T11:24:05.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1ca3fff700 10 client.26527 handle_command_reply: tid=0 2023-07-14T11:24:05.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.883+0000 7f1cdeb1eb80 1 client.26527 shutdown 2023-07-14T11:24:05.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 msgr2=0x55c2f6f32670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f1cb800eb00 tx=0x7f1cb801e420 comp rx=0 tx=0).stop 2023-07-14T11:24:05.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 shutdown_connections 2023-07-14T11:24:05.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c2f6f90750 0x55c2f6f92b60 crc :-1 s=READY pgs=118 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c2f6f2bd30 0x55c2f6f30e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c2f6f214f0 0x55c2f6f32670 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1ca3fff700 0 client.26527 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:05.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 --2- 172.21.15.40:0/113708450 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c2f6f21e00 0x55c2f6f30930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 >> 172.21.15.40:0/113708450 conn(0x55c2f6f16ff0 msgr2=0x55c2f6f23d90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 shutdown_connections 2023-07-14T11:24:05.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 1 -- 172.21.15.40:0/113708450 wait complete. 2023-07-14T11:24:05.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.884+0000 7f1cdeb1eb80 20 client.26527 trim_cache size 0 max 16384 2023-07-14T11:24:05.899 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:24:05.899 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:24:05.900 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:24:05.900 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:24:05.900 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:24:05.900 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:24:05.901 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:24:05.901 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:24:05.901 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:24:05.901 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:24:05.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 msgr2=0x7f1cb40a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 0x7f1cb40a3890 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f1cc4100090 tx=0x7f1cb0009260 comp rx=0 tx=0).stop 2023-07-14T11:24:05.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 msgr2=0x7f1cc419da30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:05.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f1cb800eb30 tx=0x7f1cb800ee40 comp rx=0 tx=0).stop 2023-07-14T11:24:05.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 shutdown_connections 2023-07-14T11:24:05.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f1cb40a13e0 0x7f1cb40a3890 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f1cc4106ea0 0x7f1cc4197aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1cc4101e60 0x7f1cc419da30 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 --2- 172.21.15.40:0/627745808 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1cc4102800 0x7f1cc419df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:05.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 >> 172.21.15.40:0/627745808 conn(0x7f1cc40fd690 msgr2=0x7f1cc40fdfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:05.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.885+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 shutdown_connections 2023-07-14T11:24:05.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:05.886+0000 7f1cc9f9d700 1 -- 172.21.15.40:0/627745808 wait complete. 2023-07-14T11:24:05.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: pgmap v371: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 28 op/s 2023-07-14T11:24:05.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: 761 slow requests, 5 included below; oldest blocked for > 606.816082 secs 2023-07-14T11:24:05.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: slow request 605.060450 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:05.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: slow request 605.034477 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:05.912 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: slow request 605.008353 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:05.913 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: slow request 604.981310 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:05.913 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:05 smithi040 ceph-mon[97954]: slow request 604.953235 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:05.915 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:24:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: pgmap v371: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 28 op/s 2023-07-14T11:24:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: 761 slow requests, 5 included below; oldest blocked for > 606.816082 secs 2023-07-14T11:24:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: slow request 605.060450 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: slow request 605.034477 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: slow request 605.008353 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: slow request 604.981310 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:05 smithi130 ceph-mon[129405]: slow request 604.953235 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: pgmap v371: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 28 op/s 2023-07-14T11:24:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: 761 slow requests, 5 included below; oldest blocked for > 606.816082 secs 2023-07-14T11:24:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: slow request 605.060450 seconds old, received at 2023-07-14T11:13:59.858211+0000: client_request(mds.1:2424 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: slow request 605.034477 seconds old, received at 2023-07-14T11:13:59.884185+0000: client_request(mds.1:2456 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: slow request 605.008353 seconds old, received at 2023-07-14T11:13:59.910309+0000: client_request(mds.1:2488 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: slow request 604.981310 seconds old, received at 2023-07-14T11:13:59.937352+0000: client_request(mds.1:2520 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:05 smithi045 ceph-mon[128791]: slow request 604.953235 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:07 smithi130 ceph-mon[129405]: pgmap v372: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:07 smithi040 ceph-mon[97954]: pgmap v372: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:07 smithi045 ceph-mon[128791]: pgmap v372: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: 744 slow requests, 5 included below; oldest blocked for > 607.167883 secs 2023-07-14T11:24:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: slow request 606.723920 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: slow request 606.717976 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: slow request 606.712090 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: slow request 606.706121 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:08 smithi130 ceph-mon[129405]: slow request 606.700196 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: 744 slow requests, 5 included below; oldest blocked for > 607.167883 secs 2023-07-14T11:24:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: slow request 606.723920 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: slow request 606.717976 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: slow request 606.712090 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: slow request 606.706121 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:08 smithi040 ceph-mon[97954]: slow request 606.700196 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: 744 slow requests, 5 included below; oldest blocked for > 607.167883 secs 2023-07-14T11:24:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: slow request 606.723920 seconds old, received at 2023-07-14T11:14:01.377519+0000: peer_request:mds.1:2537 currently dispatched 2023-07-14T11:24:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: slow request 606.717976 seconds old, received at 2023-07-14T11:14:01.383463+0000: peer_request:mds.1:2569 currently dispatched 2023-07-14T11:24:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: slow request 606.712090 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: slow request 606.706121 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:08 smithi045 ceph-mon[128791]: slow request 606.700196 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:09 smithi130 ceph-mon[129405]: pgmap v373: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:09 smithi130 ceph-mon[129405]: osdmap e104: 12 total, 12 up, 12 in 2023-07-14T11:24:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:09 smithi040 ceph-mon[97954]: pgmap v373: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:09 smithi040 ceph-mon[97954]: osdmap e104: 12 total, 12 up, 12 in 2023-07-14T11:24:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:09 smithi045 ceph-mon[128791]: pgmap v373: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:24:10.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:09 smithi045 ceph-mon[128791]: osdmap e104: 12 total, 12 up, 12 in 2023-07-14T11:24:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: 734 slow requests, 5 included below; oldest blocked for > 611.915050 secs 2023-07-14T11:24:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: slow request 610.052203 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: slow request 610.024307 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: slow request 609.997092 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: slow request 609.971121 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:10 smithi130 ceph-mon[129405]: slow request 609.944706 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: 734 slow requests, 5 included below; oldest blocked for > 611.915050 secs 2023-07-14T11:24:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: slow request 610.052203 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: slow request 610.024307 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: slow request 609.997092 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: slow request 609.971121 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:10 smithi040 ceph-mon[97954]: slow request 609.944706 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: 734 slow requests, 5 included below; oldest blocked for > 611.915050 secs 2023-07-14T11:24:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: slow request 610.052203 seconds old, received at 2023-07-14T11:13:59.965427+0000: client_request(mds.1:2552 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: slow request 610.024307 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: slow request 609.997092 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: slow request 609.971121 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:10 smithi045 ceph-mon[128791]: slow request 609.944706 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:11 smithi130 ceph-mon[129405]: pgmap v375: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:11 smithi040 ceph-mon[97954]: pgmap v375: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:11 smithi045 ceph-mon[128791]: pgmap v375: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:13.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: pgmap v376: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: 717 slow requests, 5 included below; oldest blocked for > 612.168027 secs 2023-07-14T11:24:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: slow request 611.712234 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: slow request 611.706265 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: slow request 611.700340 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: slow request 611.694555 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:14.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:13 smithi130 ceph-mon[129405]: slow request 611.688598 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: pgmap v376: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: 717 slow requests, 5 included below; oldest blocked for > 612.168027 secs 2023-07-14T11:24:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: slow request 611.712234 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:14.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: slow request 611.706265 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:14.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: slow request 611.700340 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:14.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: slow request 611.694555 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:14.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:13 smithi040 ceph-mon[97954]: slow request 611.688598 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: pgmap v376: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 28 op/s 2023-07-14T11:24:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: 717 slow requests, 5 included below; oldest blocked for > 612.168027 secs 2023-07-14T11:24:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: slow request 611.712234 seconds old, received at 2023-07-14T11:14:01.389349+0000: peer_request:mds.1:2601 currently dispatched 2023-07-14T11:24:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: slow request 611.706265 seconds old, received at 2023-07-14T11:14:01.395318+0000: peer_request:mds.1:2633 currently dispatched 2023-07-14T11:24:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: slow request 611.700340 seconds old, received at 2023-07-14T11:14:01.401243+0000: peer_request:mds.1:2665 currently dispatched 2023-07-14T11:24:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: slow request 611.694555 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:13 smithi045 ceph-mon[128791]: slow request 611.688598 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: osdmap e105: 12 total, 12 up, 12 in 2023-07-14T11:24:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: pgmap v378: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: 706 slow requests, 5 included below; oldest blocked for > 616.952096 secs 2023-07-14T11:24:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: slow request 615.061353 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: slow request 615.034138 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: slow request 615.008167 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: slow request 614.981752 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:15 smithi130 ceph-mon[129405]: slow request 614.954946 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: osdmap e105: 12 total, 12 up, 12 in 2023-07-14T11:24:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: pgmap v378: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: 706 slow requests, 5 included below; oldest blocked for > 616.952096 secs 2023-07-14T11:24:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: slow request 615.061353 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: slow request 615.034138 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: slow request 615.008167 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: slow request 614.981752 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:15 smithi040 ceph-mon[97954]: slow request 614.954946 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: osdmap e105: 12 total, 12 up, 12 in 2023-07-14T11:24:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: pgmap v378: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: 706 slow requests, 5 included below; oldest blocked for > 616.952096 secs 2023-07-14T11:24:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: slow request 615.061353 seconds old, received at 2023-07-14T11:13:59.993323+0000: client_request(mds.1:2584 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: slow request 615.034138 seconds old, received at 2023-07-14T11:14:00.020537+0000: client_request(mds.1:2616 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: slow request 615.008167 seconds old, received at 2023-07-14T11:14:00.046508+0000: client_request(mds.1:2648 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: slow request 614.981752 seconds old, received at 2023-07-14T11:14:00.072924+0000: client_request(mds.1:2680 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:16.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:15 smithi045 ceph-mon[128791]: slow request 614.954946 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:17 smithi130 ceph-mon[129405]: pgmap v379: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:17 smithi040 ceph-mon[97954]: pgmap v379: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:17 smithi045 ceph-mon[128791]: pgmap v379: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:24:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: 688 slow requests, 5 included below; oldest blocked for > 617.168144 secs 2023-07-14T11:24:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: slow request 616.694671 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: slow request 616.688714 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: slow request 616.682842 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: slow request 616.676972 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:18 smithi130 ceph-mon[129405]: slow request 616.671051 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: 688 slow requests, 5 included below; oldest blocked for > 617.168144 secs 2023-07-14T11:24:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: slow request 616.694671 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: slow request 616.688714 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: slow request 616.682842 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: slow request 616.676972 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:18 smithi040 ceph-mon[97954]: slow request 616.671051 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: 688 slow requests, 5 included below; oldest blocked for > 617.168144 secs 2023-07-14T11:24:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: slow request 616.694671 seconds old, received at 2023-07-14T11:14:01.407029+0000: peer_request:mds.1:2697 currently dispatched 2023-07-14T11:24:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: slow request 616.688714 seconds old, received at 2023-07-14T11:14:01.412986+0000: peer_request:mds.1:2729 currently dispatched 2023-07-14T11:24:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: slow request 616.682842 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: slow request 616.676972 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:18 smithi045 ceph-mon[128791]: slow request 616.671051 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:19 smithi130 ceph-mon[129405]: pgmap v380: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:24:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:19 smithi040 ceph-mon[97954]: pgmap v380: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:24:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:19 smithi045 ceph-mon[128791]: pgmap v380: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:24:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: 677 slow requests, 5 included below; oldest blocked for > 621.956253 secs 2023-07-14T11:24:21.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: slow request 619.959103 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: slow request 619.932999 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: slow request 619.906195 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: slow request 619.880528 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:20 smithi130 ceph-mon[129405]: slow request 619.854317 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: 677 slow requests, 5 included below; oldest blocked for > 621.956253 secs 2023-07-14T11:24:21.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: slow request 619.959103 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: slow request 619.932999 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: slow request 619.906195 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: slow request 619.880528 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:20 smithi040 ceph-mon[97954]: slow request 619.854317 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: 677 slow requests, 5 included below; oldest blocked for > 621.956253 secs 2023-07-14T11:24:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: slow request 619.959103 seconds old, received at 2023-07-14T11:14:00.099729+0000: client_request(mds.1:2712 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: slow request 619.932999 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: slow request 619.906195 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: slow request 619.880528 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:21.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:20 smithi045 ceph-mon[128791]: slow request 619.854317 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:21 smithi130 ceph-mon[129405]: pgmap v381: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:21 smithi040 ceph-mon[97954]: pgmap v381: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:21 smithi045 ceph-mon[128791]: pgmap v381: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: pgmap v382: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: 658 slow requests, 5 included below; oldest blocked for > 622.168247 secs 2023-07-14T11:24:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: slow request 621.682945 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: slow request 621.677075 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: slow request 621.671154 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: slow request 621.665340 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:24.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:23 smithi130 ceph-mon[129405]: slow request 621.777449 seconds old, received at 2023-07-14T11:14:01.324354+0000: peer_request:mds.1:2250 currently dispatched 2023-07-14T11:24:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: pgmap v382: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: 658 slow requests, 5 included below; oldest blocked for > 622.168247 secs 2023-07-14T11:24:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: slow request 621.682945 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: slow request 621.677075 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: slow request 621.671154 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: slow request 621.665340 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:24.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:23 smithi040 ceph-mon[97954]: slow request 621.777449 seconds old, received at 2023-07-14T11:14:01.324354+0000: peer_request:mds.1:2250 currently dispatched 2023-07-14T11:24:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: pgmap v382: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:24:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: 658 slow requests, 5 included below; oldest blocked for > 622.168247 secs 2023-07-14T11:24:24.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: slow request 621.682945 seconds old, received at 2023-07-14T11:14:01.418858+0000: peer_request:mds.1:2761 currently dispatched 2023-07-14T11:24:24.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: slow request 621.677075 seconds old, received at 2023-07-14T11:14:01.424728+0000: peer_request:mds.1:2793 currently dispatched 2023-07-14T11:24:24.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: slow request 621.671154 seconds old, received at 2023-07-14T11:14:01.430649+0000: peer_request:mds.1:2825 currently dispatched 2023-07-14T11:24:24.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: slow request 621.665340 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:24.068 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:23 smithi045 ceph-mon[128791]: slow request 621.777449 seconds old, received at 2023-07-14T11:14:01.324354+0000: peer_request:mds.1:2250 currently dispatched 2023-07-14T11:24:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: pgmap v383: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:24:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: 645 slow requests, 5 included below; oldest blocked for > 627.055258 secs 2023-07-14T11:24:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: slow request 625.032004 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: slow request 625.005200 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: slow request 624.979533 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: slow request 624.953322 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:25 smithi130 ceph-mon[129405]: slow request 624.924496 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: pgmap v383: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:24:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: 645 slow requests, 5 included below; oldest blocked for > 627.055258 secs 2023-07-14T11:24:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: slow request 625.032004 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: slow request 625.005200 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: slow request 624.979533 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: slow request 624.953322 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:25 smithi040 ceph-mon[97954]: slow request 624.924496 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: pgmap v383: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:24:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: 645 slow requests, 5 included below; oldest blocked for > 627.055258 secs 2023-07-14T11:24:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: slow request 625.032004 seconds old, received at 2023-07-14T11:14:00.125834+0000: client_request(mds.1:2744 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: slow request 625.005200 seconds old, received at 2023-07-14T11:14:00.152638+0000: client_request(mds.1:2776 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: slow request 624.979533 seconds old, received at 2023-07-14T11:14:00.178305+0000: client_request(mds.1:2808 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: slow request 624.953322 seconds old, received at 2023-07-14T11:14:00.204516+0000: client_request(mds.1:2840 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:26.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:25 smithi045 ceph-mon[128791]: slow request 624.924496 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:27 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:28.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:27 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:28.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:27 smithi130 ceph-mon[129405]: pgmap v384: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:27 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:27 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:28.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:27 smithi040 ceph-mon[97954]: pgmap v384: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:27 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:27 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:28.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:27 smithi045 ceph-mon[128791]: pgmap v384: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: 626 slow requests, 5 included below; oldest blocked for > 627.168335 secs 2023-07-14T11:24:29.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: slow request 626.665428 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: slow request 626.771424 seconds old, received at 2023-07-14T11:14:01.330467+0000: peer_request:mds.1:2282 currently dispatched 2023-07-14T11:24:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: slow request 626.765442 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: slow request 626.759619 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:29.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:28 smithi130 ceph-mon[129405]: slow request 626.753652 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: 626 slow requests, 5 included below; oldest blocked for > 627.168335 secs 2023-07-14T11:24:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: slow request 626.665428 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:29.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: slow request 626.771424 seconds old, received at 2023-07-14T11:14:01.330467+0000: peer_request:mds.1:2282 currently dispatched 2023-07-14T11:24:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: slow request 626.765442 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: slow request 626.759619 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:29.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:28 smithi040 ceph-mon[97954]: slow request 626.753652 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:29.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: 626 slow requests, 5 included below; oldest blocked for > 627.168335 secs 2023-07-14T11:24:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: slow request 626.665428 seconds old, received at 2023-07-14T11:14:01.436463+0000: peer_request:mds.1:2857 currently dispatched 2023-07-14T11:24:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: slow request 626.771424 seconds old, received at 2023-07-14T11:14:01.330467+0000: peer_request:mds.1:2282 currently dispatched 2023-07-14T11:24:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: slow request 626.765442 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: slow request 626.759619 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:29.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:28 smithi045 ceph-mon[128791]: slow request 626.753652 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:30.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:29 smithi130 ceph-mon[129405]: pgmap v385: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:30.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:29 smithi040 ceph-mon[97954]: pgmap v385: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:29 smithi045 ceph-mon[128791]: pgmap v385: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: 612 slow requests, 5 included below; oldest blocked for > 632.128512 secs 2023-07-14T11:24:32.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: slow request 629.997751 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: slow request 630.493365 seconds old, received at 2023-07-14T11:13:59.737727+0000: client_request(mds.1:2297 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: slow request 630.452424 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: slow request 630.424689 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: slow request 630.398102 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:31 smithi130 ceph-mon[129405]: pgmap v386: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: 612 slow requests, 5 included below; oldest blocked for > 632.128512 secs 2023-07-14T11:24:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: slow request 629.997751 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: slow request 630.493365 seconds old, received at 2023-07-14T11:13:59.737727+0000: client_request(mds.1:2297 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: slow request 630.452424 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: slow request 630.424689 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: slow request 630.398102 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:31 smithi040 ceph-mon[97954]: pgmap v386: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: 612 slow requests, 5 included below; oldest blocked for > 632.128512 secs 2023-07-14T11:24:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: slow request 629.997751 seconds old, received at 2023-07-14T11:14:00.233341+0000: client_request(mds.1:2872 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: slow request 630.493365 seconds old, received at 2023-07-14T11:13:59.737727+0000: client_request(mds.1:2297 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: slow request 630.452424 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: slow request 630.424689 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: slow request 630.398102 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:32.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:31 smithi045 ceph-mon[128791]: pgmap v386: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:33.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:32 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:33.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:32 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:33.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:32 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:33.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:32 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:33.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:32 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:33.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:32 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: pgmap v387: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: 592 slow requests, 5 included below; oldest blocked for > 632.168397 secs 2023-07-14T11:24:34.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: slow request 631.765505 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: slow request 631.759682 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: slow request 631.753714 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: slow request 631.747754 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:34.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:33 smithi045 ceph-mon[128791]: slow request 631.741880 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: pgmap v387: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: 592 slow requests, 5 included below; oldest blocked for > 632.168397 secs 2023-07-14T11:24:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: slow request 631.765505 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: slow request 631.759682 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: slow request 631.753714 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: slow request 631.747754 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:34.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:33 smithi130 ceph-mon[129405]: slow request 631.741880 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: pgmap v387: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: 592 slow requests, 5 included below; oldest blocked for > 632.168397 secs 2023-07-14T11:24:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: slow request 631.765505 seconds old, received at 2023-07-14T11:14:01.336448+0000: peer_request:mds.1:2314 currently dispatched 2023-07-14T11:24:34.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: slow request 631.759682 seconds old, received at 2023-07-14T11:14:01.342272+0000: peer_request:mds.1:2346 currently dispatched 2023-07-14T11:24:34.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: slow request 631.753714 seconds old, received at 2023-07-14T11:14:01.348239+0000: peer_request:mds.1:2378 currently dispatched 2023-07-14T11:24:34.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: slow request 631.747754 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:34.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:33 smithi040 ceph-mon[97954]: slow request 631.741880 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:35.916 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:24:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:35 smithi045 ceph-mon[128791]: pgmap v388: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:35 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:35 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:36.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 -- 172.21.15.40:0/1675561665 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80728b0 msgr2=0x7fc3d8072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 --2- 172.21.15.40:0/1675561665 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80728b0 0x7fc3d8072c80 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7fc3c8009990 tx=0x7fc3c8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 -- 172.21.15.40:0/1675561665 shutdown_connections 2023-07-14T11:24:36.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 --2- 172.21.15.40:0/1675561665 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 0x7fc3d8112ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 --2- 172.21.15.40:0/1675561665 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80728b0 0x7fc3d8072c80 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 --2- 172.21.15.40:0/1675561665 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80731c0 0x7fc3d81105a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 -- 172.21.15.40:0/1675561665 >> 172.21.15.40:0/1675561665 conn(0x7fc3d80fb5f0 msgr2=0x7fc3d80fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 -- 172.21.15.40:0/1675561665 shutdown_connections 2023-07-14T11:24:36.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.124+0000 7fc3dddff700 1 -- 172.21.15.40:0/1675561665 wait complete. 2023-07-14T11:24:36.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.125+0000 7fc3dddff700 1 Processor -- start 2023-07-14T11:24:36.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.125+0000 7fc3dddff700 1 -- start start 2023-07-14T11:24:36.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 0x7fc3d819c2b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 0x7fc3d81a0c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d81a1290 con 0x7fc3d80728b0 2023-07-14T11:24:36.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d81a1400 con 0x7fc3d8110ae0 2023-07-14T11:24:36.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3dddff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc3d81a15c0 con 0x7fc3d80731c0 2023-07-14T11:24:36.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3d77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 0x7fc3d819c2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3d6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3d6ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59260/0 (socket says 172.21.15.40:59260) 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3d6ffd700 1 -- 172.21.15.40:0/4213244205 learned_addr learned my addr 172.21.15.40:0/4213244205 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.126+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 0x7fc3d81a0c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 -- 172.21.15.40:0/4213244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 msgr2=0x7fc3d81a0c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 0x7fc3d81a0c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 -- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 msgr2=0x7fc3d819c2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 0x7fc3d819c2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 -- 172.21.15.40:0/4213244205 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc3cc009520 con 0x7fc3d80731c0 2023-07-14T11:24:36.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 0x7fc3d819c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:24:36.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc00eb30 tx=0x7fc3cc00ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d4ff9700 1 -- 172.21.15.40:0/4213244205 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3cc009880 con 0x7fc3d80731c0 2023-07-14T11:24:36.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d4ff9700 1 -- 172.21.15.40:0/4213244205 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3cc0044e0 con 0x7fc3d80731c0 2023-07-14T11:24:36.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc3c80095f0 con 0x7fc3d80731c0 2023-07-14T11:24:36.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.127+0000 7fc3d4ff9700 1 -- 172.21.15.40:0/4213244205 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3cc003d80 con 0x7fc3d80731c0 2023-07-14T11:24:36.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.128+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3d81a8f00 con 0x7fc3d80731c0 2023-07-14T11:24:36.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.132+0000 7fc3f1645b80 1 Processor -- start 2023-07-14T11:24:36.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.133+0000 7fc3d4ff9700 1 -- 172.21.15.40:0/4213244205 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fc3cc003f20 con 0x7fc3d80731c0 2023-07-14T11:24:36.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.133+0000 7fc3f1645b80 1 -- start start 2023-07-14T11:24:36.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a3c590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a3cad0 0x559e51a3eeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51937cb0 con 0x559e51a3cad0 2023-07-14T11:24:36.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e5198aa40 con 0x559e51a312d0 2023-07-14T11:24:36.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51983ed0 con 0x559e51a31be0 2023-07-14T11:24:36.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34726/0 (socket says 172.21.15.40:34726) 2023-07-14T11:24:36.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3d7fff700 1 -- 172.21.15.40:0/3776412726 learned_addr learned my addr 172.21.15.40:0/3776412726 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:36.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a3c590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.134+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a3cad0 0x559e51a3eeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 -- 172.21.15.40:0/3776412726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 msgr2=0x559e51a3c590 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a3c590 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 -- 172.21.15.40:0/3776412726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a3cad0 msgr2=0x559e51a3eeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a3cad0 0x559e51a3eeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 -- 172.21.15.40:0/3776412726 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559e517b1de0 con 0x559e51a312d0 2023-07-14T11:24:36.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a3c590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:24:36.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0011600 tx=0x7fc3d0009920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=829c042c973e4be1 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d4ff9700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 0x7fc3c00a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3bdffb700 1 -- 172.21.15.40:0/3776412726 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3d000e040 con 0x559e51a312d0 2023-07-14T11:24:36.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.135+0000 7fc3d4ff9700 1 -- 172.21.15.40:0/4213244205 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fc3cc014070 con 0x7fc3d80731c0 2023-07-14T11:24:36.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3bdffb700 1 -- 172.21.15.40:0/3776412726 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3d000f040 con 0x559e51a312d0 2023-07-14T11:24:36.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3bdffb700 1 -- 172.21.15.40:0/3776412726 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3d0004760 con 0x559e51a312d0 2023-07-14T11:24:36.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 0x7fc3c00a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 -- 172.21.15.40:0/3776412726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 msgr2=0x559e51a316a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0011600 tx=0x7fc3d0009920 comp rx=0 tx=0).stop 2023-07-14T11:24:36.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 -- 172.21.15.40:0/3776412726 shutdown_connections 2023-07-14T11:24:36.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a3cad0 0x559e51a3eeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a316a0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/3776412726 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a3c590 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.136+0000 7fc3f1645b80 1 -- 172.21.15.40:0/3776412726 >> 172.21.15.40:0/3776412726 conn(0x559e51a27da0 msgr2=0x559e51a2d6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.137+0000 7fc3f1645b80 1 -- 172.21.15.40:0/3776412726 shutdown_connections 2023-07-14T11:24:36.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.137+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 0x7fc3c00a3770 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fc3c800b2e0 tx=0x7fc3c8005fd0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.137+0000 7fc3f1645b80 1 -- 172.21.15.40:0/3776412726 wait complete. 2023-07-14T11:24:36.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.137+0000 7fc3f1645b80 1 -- start start 2023-07-14T11:24:36.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.137+0000 7fc3f1645b80 1 Processor -- start 2023-07-14T11:24:36.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 0x559e51a3c540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a397c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51923ac0 con 0x559e51a31be0 2023-07-14T11:24:36.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e519237c0 con 0x559e51a3cad0 2023-07-14T11:24:36.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51807920 con 0x559e51a312d0 2023-07-14T11:24:36.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3f1645b80 10 client.0 _ll_register_callbacks cb 0x559e51921760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:24:36.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3d77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 0x559e51a3c540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3d77fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 0x559e51a3c540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:59270/0 (socket says 172.21.15.40:59270) 2023-07-14T11:24:36.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3d77fe700 1 -- 172.21.15.40:0/508038584 learned_addr learned my addr 172.21.15.40:0/508038584 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:36.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.138+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34728/0 (socket says 172.21.15.40:34728) 2023-07-14T11:24:36.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a397c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 -- 172.21.15.40:0/508038584 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 msgr2=0x559e51a3c540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 0x559e51a3c540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 -- 172.21.15.40:0/508038584 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 msgr2=0x559e51a397c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a397c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 -- 172.21.15.40:0/508038584 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3d8113780 con 0x559e51a3cad0 2023-07-14T11:24:36.151 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0006080 tx=0x7fc3d00045b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=db2d44f205613543 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3b6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:24:36.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3d000e030 con 0x559e51a3cad0 2023-07-14T11:24:36.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x559e518dd4f0 con 0x559e51a3cad0 2023-07-14T11:24:36.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.139+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fc3d00092e0 con 0x559e51a3cad0 2023-07-14T11:24:36.152 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=231) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3d00040c0 con 0x559e51a3cad0 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 10 client.26573 fetch_fsmap learned FSMap version 231 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=231,monmap=4+}) v3 -- 0x7fc3d00112b0 con 0x559e51a3cad0 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x559e518dd4f0 con 0x559e51a3cad0 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 231) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fc3d0024020 con 0x559e51a3cad0 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 10 client.26573 fetch_fsmap finished waiting for FSMap version 231 2023-07-14T11:24:36.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 10 client.26573 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:24:36.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 20 client.26573 populate_metadata read hostname 'smithi040' 2023-07-14T11:24:36.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51a4c270 0x559e51a4e650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 4 client.26573 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:24:36.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x559e5197ceb0 con 0x559e51a4c270 2023-07-14T11:24:36.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.140+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51a4c270 0x559e51a4e650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.141+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51a4c270 0x559e51a4e650 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3045d08d2de72492 server_cookie=cabac81b2e67093e in_seq=0 out_seq=0 2023-07-14T11:24:36.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.141+0000 7fc3b6ffd700 10 client.26573 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:36.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/508038584 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x559e5197ceb0 con 0x559e51a4c270 2023-07-14T11:24:36.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3b6ffd700 10 client.26573 handle_command_reply: tid=0 2023-07-14T11:24:36.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 client.26573 shutdown 2023-07-14T11:24:36.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 msgr2=0x559e51a39eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0006080 tx=0x7fc3d00045b0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 shutdown_connections 2023-07-14T11:24:36.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51a4c270 0x559e51a4e650 crc :-1 s=READY pgs=119 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a39eb0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a312d0 0x559e51a3c540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3b6ffd700 0 client.26573 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:36.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/508038584 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a397c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 >> 172.21.15.40:0/508038584 conn(0x559e51a27da0 msgr2=0x559e51a2d230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 shutdown_connections 2023-07-14T11:24:36.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.142+0000 7fc3f1645b80 1 -- 172.21.15.40:0/508038584 wait complete. 2023-07-14T11:24:36.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.143+0000 7fc3f1645b80 20 client.26573 trim_cache size 0 max 16384 2023-07-14T11:24:36.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.160+0000 7fc3f1645b80 1 Processor -- start 2023-07-14T11:24:36.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.160+0000 7fc3f1645b80 1 -- start start 2023-07-14T11:24:36.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a3c070 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 0x559e51a34b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51a32540 con 0x559e51a31be0 2023-07-14T11:24:36.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51a3a490 con 0x559e51a312d0 2023-07-14T11:24:36.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51a3ac50 con 0x559e51a3cad0 2023-07-14T11:24:36.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3d77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3d77fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34738/0 (socket says 172.21.15.40:34738) 2023-07-14T11:24:36.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3d77fe700 1 -- 172.21.15.40:0/944072068 learned_addr learned my addr 172.21.15.40:0/944072068 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:36.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.161+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 0x559e51a34b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 -- 172.21.15.40:0/944072068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 msgr2=0x559e51a34b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 0x559e51a34b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 -- 172.21.15.40:0/944072068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 msgr2=0x559e51a3c070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a3c070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 -- 172.21.15.40:0/944072068 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x559e5179ac60 con 0x559e51a312d0 2023-07-14T11:24:36.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 0x559e51a34b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:24:36.170 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fc3c8000c00 tx=0x7fc3c8023000 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e009b6503b1d5b11 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3be7fc700 1 -- 172.21.15.40:0/944072068 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3c8037360 con 0x559e51a312d0 2023-07-14T11:24:36.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3be7fc700 1 -- 172.21.15.40:0/944072068 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3c801e040 con 0x559e51a312d0 2023-07-14T11:24:36.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.162+0000 7fc3be7fc700 1 -- 172.21.15.40:0/944072068 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3c800f6c0 con 0x559e51a312d0 2023-07-14T11:24:36.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- 172.21.15.40:0/944072068 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 msgr2=0x559e51a342f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fc3c8000c00 tx=0x7fc3c8023000 comp rx=0 tx=0).stop 2023-07-14T11:24:36.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- 172.21.15.40:0/944072068 shutdown_connections 2023-07-14T11:24:36.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a3cad0 0x559e51a34b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a312d0 0x559e51a342f0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/944072068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a31be0 0x559e51a3c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- 172.21.15.40:0/944072068 >> 172.21.15.40:0/944072068 conn(0x559e51a27da0 msgr2=0x559e51a2be10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- 172.21.15.40:0/944072068 shutdown_connections 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- 172.21.15.40:0/944072068 wait complete. 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.163+0000 7fc3f1645b80 1 -- start start 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 Processor -- start 2023-07-14T11:24:36.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 0x559e51a43460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a41750 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51a42360 con 0x559e51a312d0 2023-07-14T11:24:36.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51a424d0 con 0x559e51a3cad0 2023-07-14T11:24:36.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x559e51aa0110 con 0x559e51a31be0 2023-07-14T11:24:36.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.164+0000 7fc3f1645b80 10 client.0 _ll_register_callbacks cb 0x7fc3cc005ec0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:24:36.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34742/0 (socket says 172.21.15.40:34742) 2023-07-14T11:24:36.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 -- 172.21.15.40:0/274045209 learned_addr learned my addr 172.21.15.40:0/274045209 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:24:36.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 0x559e51a43460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 -- 172.21.15.40:0/274045209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 msgr2=0x559e51a41750 unknown :-1 s=STATE_CONNECTING_RE l=0).mark_down 2023-07-14T11:24:36.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a41750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a41750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 -- 172.21.15.40:0/274045209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 msgr2=0x559e51a43460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:24:36.176 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 0x559e51a43460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d7fff700 1 -- 172.21.15.40:0/274045209 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc3d00112b0 con 0x559e51a3cad0 2023-07-14T11:24:36.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 0x559e51a43460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:24:36.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.165+0000 7fc3d6ffd700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a41750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2023-07-14T11:24:36.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3d7fff700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0009ec0 tx=0x7fc3d0009060 comp rx=0 tx=0).ready entity=mon.2 client_cookie=71da38d7142c302d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:24:36.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3b6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:24:36.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc3d000e040 con 0x559e51a3cad0 2023-07-14T11:24:36.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fc3d000f040 con 0x559e51a3cad0 2023-07-14T11:24:36.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fc3d0007da0 con 0x559e51a3cad0 2023-07-14T11:24:36.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=231) v2 ==== 24+0+0 (secure 0 0 0) 0x7fc3d0004c70 con 0x559e51a3cad0 2023-07-14T11:24:36.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3f1645b80 10 client.26579 fetch_fsmap learned FSMap version 231 2023-07-14T11:24:36.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=231,monmap=4+}) v3 -- 0x7fc3c8005600 con 0x559e51a3cad0 2023-07-14T11:24:36.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.166+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc3d000f040 con 0x559e51a3cad0 2023-07-14T11:24:36.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 231) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fc3d0024050 con 0x559e51a3cad0 2023-07-14T11:24:36.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 10 client.26579 fetch_fsmap finished waiting for FSMap version 231 2023-07-14T11:24:36.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 10 client.26579 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:24:36.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 20 client.26579 populate_metadata read hostname 'smithi040' 2023-07-14T11:24:36.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51aa1190 0x559e51aa3570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:24:36.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 4 client.26579 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:24:36.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x559e5193f050 con 0x559e51aa1190 2023-07-14T11:24:36.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51aa1190 0x559e51aa3570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:24:36.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3d77fe700 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51aa1190 0x559e51aa3570 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=89ff63ae13d73364 server_cookie=7ed8220243e2e694 in_seq=0 out_seq=0 2023-07-14T11:24:36.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.167+0000 7fc3b6ffd700 10 client.26579 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:36.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3b6ffd700 1 -- 172.21.15.40:0/274045209 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x559e5193f050 con 0x559e51aa1190 2023-07-14T11:24:36.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3b6ffd700 10 client.26579 handle_command_reply: tid=0 2023-07-14T11:24:36.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 client.26579 shutdown 2023-07-14T11:24:36.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 msgr2=0x559e51a41c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fc3d0009ec0 tx=0x7fc3d0009060 comp rx=0 tx=0).stop 2023-07-14T11:24:36.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 shutdown_connections 2023-07-14T11:24:36.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x559e51aa1190 0x559e51aa3570 crc :-1 s=READY pgs=120 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x559e51a3cad0 0x559e51a41c90 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x559e51a312d0 0x559e51a43460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 --2- 172.21.15.40:0/274045209 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x559e51a31be0 0x559e51a41750 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3b6ffd700 0 client.26579 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 >> 172.21.15.40:0/274045209 conn(0x559e51a27da0 msgr2=0x559e51a34b30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 shutdown_connections 2023-07-14T11:24:36.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.168+0000 7fc3f1645b80 1 -- 172.21.15.40:0/274045209 wait complete. 2023-07-14T11:24:36.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3f1645b80 20 client.26579 trim_cache size 0 max 16384 2023-07-14T11:24:36.187 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:24:36.188 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:24:36.188 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:24:36.188 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:24:36.188 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:24:36.188 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:24:36.189 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:24:36.189 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:24:36.189 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:24:36.189 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:24:36.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 msgr2=0x7fc3c00a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 0x7fc3c00a3770 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7fc3c800b2e0 tx=0x7fc3c8005fd0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 msgr2=0x7fc3d819c7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:24:36.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7fc3cc00eb30 tx=0x7fc3cc00ee40 comp rx=0 tx=0).stop 2023-07-14T11:24:36.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 shutdown_connections 2023-07-14T11:24:36.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc3c00a12c0 0x7fc3c00a3770 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc3d8110ae0 0x7fc3d81a0c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc3d80728b0 0x7fc3d819c2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 --2- 172.21.15.40:0/4213244205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc3d80731c0 0x7fc3d819c7f0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:24:36.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 >> 172.21.15.40:0/4213244205 conn(0x7fc3d80fb5f0 msgr2=0x7fc3d80fc250 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:24:36.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 shutdown_connections 2023-07-14T11:24:36.192 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:24:36.169+0000 7fc3dddff700 1 -- 172.21.15.40:0/4213244205 wait complete. 2023-07-14T11:24:36.192 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:35 smithi040 ceph-mon[97954]: pgmap v388: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:36.193 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:35 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:36.193 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:35 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:36.194 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:24:36.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:35 smithi130 ceph-mon[129405]: pgmap v388: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:36.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:35 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:36.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:35 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:37.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: 578 slow requests, 5 included below; oldest blocked for > 637.144429 secs 2023-07-14T11:24:37.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: slow request 635.468340 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: slow request 635.440606 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: slow request 635.414018 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: slow request 635.387996 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: slow request 635.362021 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:37.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:36 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: 578 slow requests, 5 included below; oldest blocked for > 637.144429 secs 2023-07-14T11:24:37.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: slow request 635.468340 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: slow request 635.440606 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: slow request 635.414018 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: slow request 635.387996 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: slow request 635.362021 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:37.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:36 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: 578 slow requests, 5 included below; oldest blocked for > 637.144429 secs 2023-07-14T11:24:37.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: slow request 635.468340 seconds old, received at 2023-07-14T11:13:59.778668+0000: client_request(mds.1:2329 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: slow request 635.440606 seconds old, received at 2023-07-14T11:13:59.806403+0000: client_request(mds.1:2361 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: slow request 635.414018 seconds old, received at 2023-07-14T11:13:59.832990+0000: client_request(mds.1:2393 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: slow request 635.387996 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: slow request 635.362021 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:37.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:37.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:36 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:37 smithi045 ceph-mon[128791]: pgmap v389: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:37 smithi130 ceph-mon[129405]: pgmap v389: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:38.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:37 smithi040 ceph-mon[97954]: pgmap v389: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:39.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: 557 slow requests, 5 included below; oldest blocked for > 637.168491 secs 2023-07-14T11:24:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: slow request 636.747848 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: slow request 636.741974 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: slow request 636.736089 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:39.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: slow request 636.730247 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:39.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:38 smithi045 ceph-mon[128791]: slow request 636.724316 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:39.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: 557 slow requests, 5 included below; oldest blocked for > 637.168491 secs 2023-07-14T11:24:39.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: slow request 636.747848 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:39.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: slow request 636.741974 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:39.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: slow request 636.736089 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:39.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: slow request 636.730247 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:39.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:38 smithi130 ceph-mon[129405]: slow request 636.724316 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:39.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: 557 slow requests, 5 included below; oldest blocked for > 637.168491 secs 2023-07-14T11:24:39.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: slow request 636.747848 seconds old, received at 2023-07-14T11:14:01.354199+0000: peer_request:mds.1:2410 currently dispatched 2023-07-14T11:24:39.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: slow request 636.741974 seconds old, received at 2023-07-14T11:14:01.360073+0000: peer_request:mds.1:2442 currently dispatched 2023-07-14T11:24:39.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: slow request 636.736089 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:39.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: slow request 636.730247 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:39.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:38 smithi040 ceph-mon[97954]: slow request 636.724316 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:39 smithi045 ceph-mon[128791]: pgmap v390: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:39 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:40.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:39 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:39 smithi130 ceph-mon[129405]: pgmap v390: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:39 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:40.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:39 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:39 smithi040 ceph-mon[97954]: pgmap v390: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:39 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:40.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:39 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: 542 slow requests, 5 included below; oldest blocked for > 642.178440 secs 2023-07-14T11:24:42.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: slow request 640.422006 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: slow request 640.396032 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: slow request 640.369898 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: slow request 640.342753 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: slow request 640.314712 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:42.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:42.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:41 smithi130 ceph-mon[129405]: pgmap v391: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: 542 slow requests, 5 included below; oldest blocked for > 642.178440 secs 2023-07-14T11:24:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: slow request 640.422006 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: slow request 640.396032 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: slow request 640.369898 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: slow request 640.342753 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: slow request 640.314712 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:42.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:42.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:41 smithi040 ceph-mon[97954]: pgmap v391: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: 542 slow requests, 5 included below; oldest blocked for > 642.178440 secs 2023-07-14T11:24:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: slow request 640.422006 seconds old, received at 2023-07-14T11:13:59.859013+0000: client_request(mds.1:2425 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: slow request 640.396032 seconds old, received at 2023-07-14T11:13:59.884987+0000: client_request(mds.1:2457 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: slow request 640.369898 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: slow request 640.342753 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: slow request 640.314712 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:42.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:42.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:41 smithi045 ceph-mon[128791]: pgmap v391: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:44.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: pgmap v392: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: 523 slow requests, 5 included below; oldest blocked for > 642.168609 secs 2023-07-14T11:24:44.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: slow request 641.736207 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: slow request 641.730365 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: slow request 641.724435 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: slow request 641.718540 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:44.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:43 smithi130 ceph-mon[129405]: slow request 641.712640 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:44.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: pgmap v392: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: 523 slow requests, 5 included below; oldest blocked for > 642.168609 secs 2023-07-14T11:24:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: slow request 641.736207 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: slow request 641.730365 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: slow request 641.724435 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:44.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: slow request 641.718540 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:44.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:43 smithi040 ceph-mon[97954]: slow request 641.712640 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: pgmap v392: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: 523 slow requests, 5 included below; oldest blocked for > 642.168609 secs 2023-07-14T11:24:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: slow request 641.736207 seconds old, received at 2023-07-14T11:14:01.365958+0000: peer_request:mds.1:2474 currently dispatched 2023-07-14T11:24:44.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: slow request 641.730365 seconds old, received at 2023-07-14T11:14:01.371800+0000: peer_request:mds.1:2506 currently dispatched 2023-07-14T11:24:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: slow request 641.724435 seconds old, received at 2023-07-14T11:14:01.377731+0000: peer_request:mds.1:2538 currently dispatched 2023-07-14T11:24:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: slow request 641.718540 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:44.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:43 smithi045 ceph-mon[128791]: slow request 641.712640 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:45.618 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:45.618 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:45.618 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: pgmap v393: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:45.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: 505 slow requests, 5 included below; oldest blocked for > 647.249785 secs 2023-07-14T11:24:45.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: slow request 645.441243 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:45.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: slow request 645.414098 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:45.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: slow request 645.386057 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:45.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: slow request 645.358102 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:45.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:45 smithi130 ceph-mon[129405]: slow request 645.331053 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: pgmap v393: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: 505 slow requests, 5 included below; oldest blocked for > 647.249785 secs 2023-07-14T11:24:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: slow request 645.441243 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: slow request 645.414098 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: slow request 645.386057 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: slow request 645.358102 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:45 smithi040 ceph-mon[97954]: slow request 645.331053 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: pgmap v393: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: 505 slow requests, 5 included below; oldest blocked for > 647.249785 secs 2023-07-14T11:24:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: slow request 645.441243 seconds old, received at 2023-07-14T11:13:59.911121+0000: client_request(mds.1:2489 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: slow request 645.414098 seconds old, received at 2023-07-14T11:13:59.938266+0000: client_request(mds.1:2521 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: slow request 645.386057 seconds old, received at 2023-07-14T11:13:59.966307+0000: client_request(mds.1:2553 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: slow request 645.358102 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:46.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:45 smithi045 ceph-mon[128791]: slow request 645.331053 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:47 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:47 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:48.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:47 smithi130 ceph-mon[129405]: pgmap v394: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:47 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:47 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:48.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:47 smithi040 ceph-mon[97954]: pgmap v394: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:47 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:47 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:48.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:47 smithi045 ceph-mon[128791]: pgmap v394: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: 483 slow requests, 5 included below; oldest blocked for > 647.168757 secs 2023-07-14T11:24:49.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: slow request 646.718688 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: slow request 646.712788 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: slow request 646.706788 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:49.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: slow request 646.700898 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:49.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:48 smithi130 ceph-mon[129405]: slow request 646.695098 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: 483 slow requests, 5 included below; oldest blocked for > 647.168757 secs 2023-07-14T11:24:49.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: slow request 646.718688 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: slow request 646.712788 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: slow request 646.706788 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:49.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: slow request 646.700898 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:49.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:48 smithi040 ceph-mon[97954]: slow request 646.695098 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:49.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: 483 slow requests, 5 included below; oldest blocked for > 647.168757 secs 2023-07-14T11:24:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: slow request 646.718688 seconds old, received at 2023-07-14T11:14:01.383625+0000: peer_request:mds.1:2570 currently dispatched 2023-07-14T11:24:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: slow request 646.712788 seconds old, received at 2023-07-14T11:14:01.389525+0000: peer_request:mds.1:2602 currently dispatched 2023-07-14T11:24:49.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: slow request 646.706788 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:49.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: slow request 646.700898 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:49.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:48 smithi045 ceph-mon[128791]: slow request 646.695098 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:50.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:49 smithi130 ceph-mon[129405]: pgmap v395: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:50.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:49 smithi040 ceph-mon[97954]: pgmap v395: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:49 smithi045 ceph-mon[128791]: pgmap v395: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:51.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: 467 slow requests, 5 included below; oldest blocked for > 652.291635 secs 2023-07-14T11:24:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: slow request 650.399953 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: slow request 650.372903 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: slow request 650.346877 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: slow request 650.320489 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:50 smithi130 ceph-mon[129405]: slow request 650.293674 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: 467 slow requests, 5 included below; oldest blocked for > 652.291635 secs 2023-07-14T11:24:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: slow request 650.399953 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: slow request 650.372903 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: slow request 650.346877 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: slow request 650.320489 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:50 smithi040 ceph-mon[97954]: slow request 650.293674 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: 467 slow requests, 5 included below; oldest blocked for > 652.291635 secs 2023-07-14T11:24:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: slow request 650.399953 seconds old, received at 2023-07-14T11:13:59.994262+0000: client_request(mds.1:2585 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: slow request 650.372903 seconds old, received at 2023-07-14T11:14:00.021311+0000: client_request(mds.1:2617 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: slow request 650.346877 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: slow request 650.320489 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:51.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:50 smithi045 ceph-mon[128791]: slow request 650.293674 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:52.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:51 smithi130 ceph-mon[129405]: pgmap v396: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:52.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:51 smithi040 ceph-mon[97954]: pgmap v396: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:51 smithi045 ceph-mon[128791]: pgmap v396: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:54.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: pgmap v397: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: 443 slow requests, 5 included below; oldest blocked for > 652.168833 secs 2023-07-14T11:24:54.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: slow request 651.706864 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: slow request 651.700974 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: slow request 651.695174 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: slow request 651.689220 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:54.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:53 smithi130 ceph-mon[129405]: slow request 651.683344 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:54.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: pgmap v397: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: 443 slow requests, 5 included below; oldest blocked for > 652.168833 secs 2023-07-14T11:24:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: slow request 651.706864 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:54.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: slow request 651.700974 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: slow request 651.695174 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: slow request 651.689220 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:54.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:53 smithi040 ceph-mon[97954]: slow request 651.683344 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:54.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: pgmap v397: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:54.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: 443 slow requests, 5 included below; oldest blocked for > 652.168833 secs 2023-07-14T11:24:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: slow request 651.706864 seconds old, received at 2023-07-14T11:14:01.395525+0000: peer_request:mds.1:2634 currently dispatched 2023-07-14T11:24:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: slow request 651.700974 seconds old, received at 2023-07-14T11:14:01.401415+0000: peer_request:mds.1:2666 currently dispatched 2023-07-14T11:24:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: slow request 651.695174 seconds old, received at 2023-07-14T11:14:01.407215+0000: peer_request:mds.1:2698 currently dispatched 2023-07-14T11:24:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: slow request 651.689220 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:54.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:53 smithi045 ceph-mon[128791]: slow request 651.683344 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: pgmap v398: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: 421 slow requests, 5 included below; oldest blocked for > 657.339488 secs 2023-07-14T11:24:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: slow request 655.394730 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: slow request 655.368342 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: slow request 655.341527 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: slow request 655.315451 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:55 smithi130 ceph-mon[129405]: slow request 655.288657 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: pgmap v398: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: 421 slow requests, 5 included below; oldest blocked for > 657.339488 secs 2023-07-14T11:24:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: slow request 655.394730 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: slow request 655.368342 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: slow request 655.341527 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: slow request 655.315451 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:55 smithi040 ceph-mon[97954]: slow request 655.288657 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:24:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: pgmap v398: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:24:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: 421 slow requests, 5 included below; oldest blocked for > 657.339488 secs 2023-07-14T11:24:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: slow request 655.394730 seconds old, received at 2023-07-14T11:14:00.047338+0000: client_request(mds.1:2649 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: slow request 655.368342 seconds old, received at 2023-07-14T11:14:00.073726+0000: client_request(mds.1:2681 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: slow request 655.341527 seconds old, received at 2023-07-14T11:14:00.100540+0000: client_request(mds.1:2713 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: slow request 655.315451 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:56.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:55 smithi045 ceph-mon[128791]: slow request 655.288657 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:24:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:57 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:58.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:57 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:58.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:57 smithi130 ceph-mon[129405]: pgmap v399: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:57 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:58.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:57 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:58.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:57 smithi040 ceph-mon[97954]: pgmap v399: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:57 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:24:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:57 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:24:58.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:57 smithi045 ceph-mon[128791]: pgmap v399: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:24:59.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:24:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: 394 slow requests, 5 included below; oldest blocked for > 657.168969 secs 2023-07-14T11:24:59.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: slow request 656.689356 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: slow request 656.683481 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: slow request 656.677615 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:24:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: slow request 656.671716 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:24:59.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: slow request 656.665860 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:24:59.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.046 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:58 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:24:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: 394 slow requests, 5 included below; oldest blocked for > 657.168969 secs 2023-07-14T11:24:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: slow request 656.689356 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:59.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: slow request 656.683481 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: slow request 656.677615 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:24:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: slow request 656.671716 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:24:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: slow request 656.665860 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:24:59.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:58 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:24:59.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: 394 slow requests, 5 included below; oldest blocked for > 657.168969 secs 2023-07-14T11:24:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: slow request 656.689356 seconds old, received at 2023-07-14T11:14:01.413169+0000: peer_request:mds.1:2730 currently dispatched 2023-07-14T11:24:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: slow request 656.683481 seconds old, received at 2023-07-14T11:14:01.419045+0000: peer_request:mds.1:2762 currently dispatched 2023-07-14T11:24:59.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: slow request 656.677615 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:24:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: slow request 656.671716 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:24:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: slow request 656.665860 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:24:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:24:59.068 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:58 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:25:00.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: pgmap v400: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:00.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:25:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:25:00.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:24:59 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:25:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:00.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: pgmap v400: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:25:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:25:00.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:24:59 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:25:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: pgmap v400: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:25:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:25:00.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:24:59 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:25:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: 372 slow requests, 5 included below; oldest blocked for > 662.370509 secs 2023-07-14T11:25:01.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: slow request 660.346472 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: slow request 660.319678 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: slow request 660.293952 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: slow request 660.267770 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: slow request 660.238729 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:01.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:00 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: 372 slow requests, 5 included below; oldest blocked for > 662.370509 secs 2023-07-14T11:25:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: slow request 660.346472 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: slow request 660.319678 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: slow request 660.293952 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: slow request 660.267770 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: slow request 660.238729 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:01.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:00 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: 372 slow requests, 5 included below; oldest blocked for > 662.370509 secs 2023-07-14T11:25:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: slow request 660.346472 seconds old, received at 2023-07-14T11:14:00.126616+0000: client_request(mds.1:2745 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: slow request 660.319678 seconds old, received at 2023-07-14T11:14:00.153411+0000: client_request(mds.1:2777 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: slow request 660.293952 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: slow request 660.267770 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: slow request 660.238729 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:01.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:00 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:02.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:01 smithi130 ceph-mon[129405]: pgmap v401: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:01 smithi040 ceph-mon[97954]: pgmap v401: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:02.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:01 smithi045 ceph-mon[128791]: pgmap v401: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:04.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: pgmap v402: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: 348 slow requests, 5 included below; oldest blocked for > 662.169067 secs 2023-07-14T11:25:04.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: slow request 661.677712 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:25:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: slow request 661.671813 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:25:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: slow request 661.665958 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:25:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: slow request 661.724722 seconds old, received at 2023-07-14T11:14:01.377901+0000: peer_request:mds.1:2539 currently dispatched 2023-07-14T11:25:04.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:03 smithi130 ceph-mon[129405]: slow request 661.718834 seconds old, received at 2023-07-14T11:14:01.383789+0000: peer_request:mds.1:2571 currently dispatched 2023-07-14T11:25:04.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: pgmap v402: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: 348 slow requests, 5 included below; oldest blocked for > 662.169067 secs 2023-07-14T11:25:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: slow request 661.677712 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:25:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: slow request 661.671813 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:25:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: slow request 661.665958 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:25:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: slow request 661.724722 seconds old, received at 2023-07-14T11:14:01.377901+0000: peer_request:mds.1:2539 currently dispatched 2023-07-14T11:25:04.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:03 smithi040 ceph-mon[97954]: slow request 661.718834 seconds old, received at 2023-07-14T11:14:01.383789+0000: peer_request:mds.1:2571 currently dispatched 2023-07-14T11:25:04.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: pgmap v402: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: 348 slow requests, 5 included below; oldest blocked for > 662.169067 secs 2023-07-14T11:25:04.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: slow request 661.677712 seconds old, received at 2023-07-14T11:14:01.424911+0000: peer_request:mds.1:2794 currently dispatched 2023-07-14T11:25:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: slow request 661.671813 seconds old, received at 2023-07-14T11:14:01.430810+0000: peer_request:mds.1:2826 currently dispatched 2023-07-14T11:25:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: slow request 661.665958 seconds old, received at 2023-07-14T11:14:01.436665+0000: peer_request:mds.1:2858 currently dispatched 2023-07-14T11:25:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: slow request 661.724722 seconds old, received at 2023-07-14T11:14:01.377901+0000: peer_request:mds.1:2539 currently dispatched 2023-07-14T11:25:04.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:03 smithi045 ceph-mon[128791]: slow request 661.718834 seconds old, received at 2023-07-14T11:14:01.383789+0000: peer_request:mds.1:2571 currently dispatched 2023-07-14T11:25:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:04 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:05.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:04 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:04 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:05.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:04 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:05.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:04 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:05.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:04 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: pgmap v403: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:25:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: 320 slow requests, 5 included below; oldest blocked for > 667.411128 secs 2023-07-14T11:25:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: slow request 665.334572 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: slow request 665.308390 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: slow request 665.279349 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: slow request 665.518490 seconds old, received at 2023-07-14T11:13:59.995218+0000: client_request(mds.1:2586 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:05 smithi130 ceph-mon[129405]: slow request 665.491586 seconds old, received at 2023-07-14T11:14:00.022122+0000: client_request(mds.1:2618 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: pgmap v403: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:25:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: 320 slow requests, 5 included below; oldest blocked for > 667.411128 secs 2023-07-14T11:25:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: slow request 665.334572 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: slow request 665.308390 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: slow request 665.279349 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: slow request 665.518490 seconds old, received at 2023-07-14T11:13:59.995218+0000: client_request(mds.1:2586 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:05 smithi040 ceph-mon[97954]: slow request 665.491586 seconds old, received at 2023-07-14T11:14:00.022122+0000: client_request(mds.1:2618 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: pgmap v403: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:25:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: 320 slow requests, 5 included below; oldest blocked for > 667.411128 secs 2023-07-14T11:25:06.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: slow request 665.334572 seconds old, received at 2023-07-14T11:14:00.179136+0000: client_request(mds.1:2809 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: slow request 665.308390 seconds old, received at 2023-07-14T11:14:00.205318+0000: client_request(mds.1:2841 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: slow request 665.279349 seconds old, received at 2023-07-14T11:14:00.234359+0000: client_request(mds.1:2873 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: slow request 665.518490 seconds old, received at 2023-07-14T11:13:59.995218+0000: client_request(mds.1:2586 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:05 smithi045 ceph-mon[128791]: slow request 665.491586 seconds old, received at 2023-07-14T11:14:00.022122+0000: client_request(mds.1:2618 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:06.194 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:25:06.408 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 -- 172.21.15.40:0/1206705834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340ff110 msgr2=0x7efc3410be70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 --2- 172.21.15.40:0/1206705834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340ff110 0x7efc3410be70 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7efc28009990 tx=0x7efc28009ca0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 -- 172.21.15.40:0/1206705834 shutdown_connections 2023-07-14T11:25:06.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 --2- 172.21.15.40:0/1206705834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 0x7efc3410e790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 --2- 172.21.15.40:0/1206705834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340fe800 0x7efc340febd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.409 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 --2- 172.21.15.40:0/1206705834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340ff110 0x7efc3410be70 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 -- 172.21.15.40:0/1206705834 >> 172.21.15.40:0/1206705834 conn(0x7efc340745d0 msgr2=0x7efc340749d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 -- 172.21.15.40:0/1206705834 shutdown_connections 2023-07-14T11:25:06.410 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.405+0000 7efc33fff700 1 -- 172.21.15.40:0/1206705834 wait complete. 2023-07-14T11:25:06.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.406+0000 7efc33fff700 1 Processor -- start 2023-07-14T11:25:06.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 -- start start 2023-07-14T11:25:06.411 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 0x7efc3419df90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 0x7efc34197a30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc34198200 con 0x7efc3410c3b0 2023-07-14T11:25:06.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc34198370 con 0x7efc340fe800 2023-07-14T11:25:06.412 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc33fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc34198530 con 0x7efc340ff110 2023-07-14T11:25:06.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.407+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34816/0 (socket says 172.21.15.40:34816) 2023-07-14T11:25:06.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 -- 172.21.15.40:0/757469544 learned_addr learned my addr 172.21.15.40:0/757469544 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:06.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc327fc700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 0x7efc3419df90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.413 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc337fe700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 0x7efc34197a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.414 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 -- 172.21.15.40:0/757469544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 msgr2=0x7efc3419df90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 0x7efc3419df90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 -- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 msgr2=0x7efc34197a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 0x7efc34197a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 -- 172.21.15.40:0/757469544 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc280095f0 con 0x7efc340fe800 2023-07-14T11:25:06.415 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc327fc700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 0x7efc3419df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:25:06.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.408+0000 7efc32ffd700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7efc2400e9a0 tx=0x7efc2400ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.409+0000 7efc1bfff700 1 -- 172.21.15.40:0/757469544 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc24009790 con 0x7efc340fe800 2023-07-14T11:25:06.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.409+0000 7efc1bfff700 1 -- 172.21.15.40:0/757469544 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc2400cbd0 con 0x7efc340fe800 2023-07-14T11:25:06.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.409+0000 7efc1bfff700 1 -- 172.21.15.40:0/757469544 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc24018680 con 0x7efc340fe800 2023-07-14T11:25:06.416 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.409+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc34198810 con 0x7efc340fe800 2023-07-14T11:25:06.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.409+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc341a49d0 con 0x7efc340fe800 2023-07-14T11:25:06.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.413+0000 7efc1bfff700 1 -- 172.21.15.40:0/757469544 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7efc2400cd40 con 0x7efc340fe800 2023-07-14T11:25:06.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.414+0000 7efc4cf9fb80 1 Processor -- start 2023-07-14T11:25:06.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.414+0000 7efc1bfff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 0x7efc1c0a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.414+0000 7efc4cf9fb80 1 -- start start 2023-07-14T11:25:06.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.414+0000 7efc1bfff700 1 -- 172.21.15.40:0/757469544 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7efc24014070 con 0x7efc340fe800 2023-07-14T11:25:06.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc337fe700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 0x7efc1c0a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d14187f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d141b110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d1313cb0 con 0x55e4d140ee00 2023-07-14T11:25:06.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d1366a40 con 0x55e4d140e4f0 2023-07-14T11:25:06.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d135fed0 con 0x55e4d1418d30 2023-07-14T11:25:06.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34826/0 (socket says 172.21.15.40:34826) 2023-07-14T11:25:06.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc32ffd700 1 -- 172.21.15.40:0/477050300 learned_addr learned my addr 172.21.15.40:0/477050300 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:06.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc337fe700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d141b110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.415+0000 7efc327fc700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d14187f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 -- 172.21.15.40:0/477050300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 msgr2=0x55e4d141b110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d141b110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 -- 172.21.15.40:0/477050300 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 msgr2=0x55e4d140e8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 -- 172.21.15.40:0/477050300 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55e4d118dde0 con 0x55e4d140ee00 2023-07-14T11:25:06.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc337fe700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d141b110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:06.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc32ffd700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:06.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc337fe700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 0x7efc1c0a3890 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7efc2c009550 tx=0x7efc2c007910 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc327fc700 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d14187f0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7efc280098e0 tx=0x7efc28005c00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3400609a398bb4cd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.416+0000 7efc197fa700 1 -- 172.21.15.40:0/477050300 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc2800beb0 con 0x55e4d140ee00 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.417+0000 7efc197fa700 1 -- 172.21.15.40:0/477050300 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc2801e040 con 0x55e4d140ee00 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.417+0000 7efc197fa700 1 -- 172.21.15.40:0/477050300 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc28004a40 con 0x55e4d140ee00 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/477050300 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 msgr2=0x55e4d14187f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d14187f0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 crypto rx=0x7efc280098e0 tx=0x7efc28005c00 comp rx=0 tx=0).stop 2023-07-14T11:25:06.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/477050300 shutdown_connections 2023-07-14T11:25:06.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d141b110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140e4f0 0x55e4d140e8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/477050300 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d14187f0 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/477050300 >> 172.21.15.40:0/477050300 conn(0x55e4d1403ff0 msgr2=0x55e4d1409930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/477050300 shutdown_connections 2023-07-14T11:25:06.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/477050300 wait complete. 2023-07-14T11:25:06.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.418+0000 7efc4cf9fb80 1 -- start start 2023-07-14T11:25:06.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 Processor -- start 2023-07-14T11:25:06.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d14186f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d1415970 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d12ffac0 con 0x55e4d140e4f0 2023-07-14T11:25:06.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d12ff7c0 con 0x55e4d140ee00 2023-07-14T11:25:06.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d11e3920 con 0x55e4d1418d30 2023-07-14T11:25:06.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc4cf9fb80 10 client.0 _ll_register_callbacks cb 0x55e4d12fd760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:25:06.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc337fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc337fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:45674/0 (socket says 172.21.15.40:45674) 2023-07-14T11:25:06.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.419+0000 7efc337fe700 1 -- 172.21.15.40:0/2364605690 learned_addr learned my addr 172.21.15.40:0/2364605690 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:06.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d14186f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc327fc700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d1415970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 -- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 msgr2=0x55e4d1415970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d1415970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 -- 172.21.15.40:0/2364605690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 msgr2=0x55e4d14186f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d14186f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 -- 172.21.15.40:0/2364605690 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc341a49d0 con 0x55e4d1418d30 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc337fe700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7efc2c019ed0 tx=0x7efc2c004b40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e4dbbd1d194e3e45 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.420+0000 7efc16ff5700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:25:06.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc2c026070 con 0x55e4d1418d30 2023-07-14T11:25:06.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55e4d12b94f0 con 0x55e4d1418d30 2023-07-14T11:25:06.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7efc2c018c60 con 0x55e4d1418d30 2023-07-14T11:25:06.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=244) v2 ==== 24+0+0 (secure 0 0 0) 0x7efc2c018e20 con 0x55e4d1418d30 2023-07-14T11:25:06.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc4cf9fb80 10 client.26554 fetch_fsmap learned FSMap version 244 2023-07-14T11:25:06.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=244,monmap=4+}) v3 -- 0x7efc2c019400 con 0x55e4d1418d30 2023-07-14T11:25:06.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55e4d12b94f0 con 0x55e4d1418d30 2023-07-14T11:25:06.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.421+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 244) v1 ==== 7470+0+0 (secure 0 0 0) 0x7efc2c029020 con 0x55e4d1418d30 2023-07-14T11:25:06.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 10 client.26554 fetch_fsmap finished waiting for FSMap version 244 2023-07-14T11:25:06.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 10 client.26554 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:25:06.436 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 20 client.26554 populate_metadata read hostname 'smithi040' 2023-07-14T11:25:06.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1430f40 0x55e4d1433350 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 4 client.26554 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:25:06.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55e4d1358eb0 con 0x55e4d1430f40 2023-07-14T11:25:06.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1430f40 0x55e4d1433350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1430f40 0x55e4d1433350 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=67e4fceeb5a0db41 server_cookie=cd7b5004ff9b13b5 in_seq=0 out_seq=0 2023-07-14T11:25:06.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.422+0000 7efc16ff5700 10 client.26554 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:06.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc16ff5700 1 -- 172.21.15.40:0/2364605690 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55e4d1358eb0 con 0x55e4d1430f40 2023-07-14T11:25:06.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc16ff5700 10 client.26554 handle_command_reply: tid=0 2023-07-14T11:25:06.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc4cf9fb80 1 client.26554 shutdown 2023-07-14T11:25:06.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 msgr2=0x55e4d1415ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7efc2c019ed0 tx=0x7efc2c004b40 comp rx=0 tx=0).stop 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 shutdown_connections 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.423+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1430f40 0x55e4d1433350 crc :-1 s=READY pgs=121 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d1415ff0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d14186f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2364605690 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d1415970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc16ff5700 0 client.26554 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:06.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 >> 172.21.15.40:0/2364605690 conn(0x55e4d1403ff0 msgr2=0x55e4d1409520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 shutdown_connections 2023-07-14T11:25:06.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2364605690 wait complete. 2023-07-14T11:25:06.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.424+0000 7efc4cf9fb80 20 client.26554 trim_cache size 0 max 16384 2023-07-14T11:25:06.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 Processor -- start 2023-07-14T11:25:06.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 -- start start 2023-07-14T11:25:06.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d1418290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 0x55e4d1412c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d140f980 con 0x55e4d140ee00 2023-07-14T11:25:06.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d1416f50 con 0x55e4d1418d30 2023-07-14T11:25:06.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d14107e0 con 0x55e4d140e4f0 2023-07-14T11:25:06.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc337fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 0x55e4d1412c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:45690/0 (socket says 172.21.15.40:45690) 2023-07-14T11:25:06.448 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.442+0000 7efc337fe700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 0x55e4d1412c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34852/0 (socket says 172.21.15.40:34852) 2023-07-14T11:25:06.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 -- 172.21.15.40:0/2602355657 learned_addr learned my addr 172.21.15.40:0/2602355657 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:06.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc327fc700 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d1418290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 -- 172.21.15.40:0/2602355657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 msgr2=0x55e4d1412c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 0x55e4d1412c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 -- 172.21.15.40:0/2602355657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 msgr2=0x55e4d1418290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d1418290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 -- 172.21.15.40:0/2602355657 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc3410f2c0 con 0x55e4d140e4f0 2023-07-14T11:25:06.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7efc24000c00 tx=0x7efc2400efa0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=223431d8189896d2 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc19ffb700 1 -- 172.21.15.40:0/2602355657 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc240b6070 con 0x55e4d140e4f0 2023-07-14T11:25:06.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc19ffb700 1 -- 172.21.15.40:0/2602355657 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc240b1070 con 0x55e4d140e4f0 2023-07-14T11:25:06.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.443+0000 7efc19ffb700 1 -- 172.21.15.40:0/2602355657 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc240cd240 con 0x55e4d140e4f0 2023-07-14T11:25:06.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2602355657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 msgr2=0x55e4d141f7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7efc24000c00 tx=0x7efc2400efa0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2602355657 shutdown_connections 2023-07-14T11:25:06.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d1418d30 0x55e4d1412c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d140e4f0 0x55e4d141f7b0 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2602355657 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140ee00 0x55e4d1418290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2602355657 >> 172.21.15.40:0/2602355657 conn(0x55e4d1403ff0 msgr2=0x55e4d1410c90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2602355657 shutdown_connections 2023-07-14T11:25:06.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.444+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2602355657 wait complete. 2023-07-14T11:25:06.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 -- start start 2023-07-14T11:25:06.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 Processor -- start 2023-07-14T11:25:06.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d147e2b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d147d000 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d147d570 con 0x55e4d140e4f0 2023-07-14T11:25:06.456 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d147d6e0 con 0x55e4d140ee00 2023-07-14T11:25:06.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.445+0000 7efc4cf9fb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55e4d147d8a0 con 0x55e4d1418d30 2023-07-14T11:25:06.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc4cf9fb80 10 client.0 _ll_register_callbacks cb 0x7efc2c0198a0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:25:06.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc32ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d147e2b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.457 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:34868/0 (socket says 172.21.15.40:34868) 2023-07-14T11:25:06.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 -- 172.21.15.40:0/2399811173 learned_addr learned my addr 172.21.15.40:0/2399811173 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:06.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc337fe700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d147d000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 -- 172.21.15.40:0/2399811173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 msgr2=0x55e4d147d000 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d147d000 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 -- 172.21.15.40:0/2399811173 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 msgr2=0x55e4d147e2b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:06.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d147e2b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc327fc700 1 -- 172.21.15.40:0/2399811173 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc34198810 con 0x55e4d140ee00 2023-07-14T11:25:06.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d147e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:06.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.446+0000 7efc337fe700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d147d000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:06.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc327fc700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7efc280098e0 tx=0x7efc28005270 comp rx=0 tx=0).ready entity=mon.2 client_cookie=ac608c6cbbc6dade server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:06.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc16ff5700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:25:06.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7efc28004500 con 0x55e4d140ee00 2023-07-14T11:25:06.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7efc2801e040 con 0x55e4d140ee00 2023-07-14T11:25:06.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(105..105 src has 1..105) v4 ==== 9432+0+0 (secure 0 0 0) 0x7efc28004660 con 0x55e4d140ee00 2023-07-14T11:25:06.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=244) v2 ==== 24+0+0 (secure 0 0 0) 0x7efc28004820 con 0x55e4d140ee00 2023-07-14T11:25:06.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc4cf9fb80 10 client.26597 fetch_fsmap learned FSMap version 244 2023-07-14T11:25:06.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.447+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=244,monmap=4+}) v3 -- 0x55e4d1176c60 con 0x55e4d140ee00 2023-07-14T11:25:06.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7efc2801e040 con 0x55e4d140ee00 2023-07-14T11:25:06.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 244) v1 ==== 7470+0+0 (secure 0 0 0) 0x7efc2802db30 con 0x55e4d140ee00 2023-07-14T11:25:06.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 10 client.26597 fetch_fsmap finished waiting for FSMap version 244 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 10 client.26597 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 20 client.26597 populate_metadata read hostname 'smithi040' 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1485f00 0x55e4d1488310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 4 client.26597 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55e4d131b050 con 0x55e4d1485f00 2023-07-14T11:25:06.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.448+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1485f00 0x55e4d1488310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:06.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc32ffd700 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1485f00 0x55e4d1488310 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=d9ccfb5159dec9f5 server_cookie=19b6e84887bc9a7c in_seq=0 out_seq=0 2023-07-14T11:25:06.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc16ff5700 10 client.26597 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:06.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc16ff5700 1 -- 172.21.15.40:0/2399811173 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55e4d131b050 con 0x55e4d1485f00 2023-07-14T11:25:06.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc16ff5700 10 client.26597 handle_command_reply: tid=0 2023-07-14T11:25:06.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 client.26597 shutdown 2023-07-14T11:25:06.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 msgr2=0x55e4d147e7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7efc280098e0 tx=0x7efc28005270 comp rx=0 tx=0).stop 2023-07-14T11:25:06.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 shutdown_connections 2023-07-14T11:25:06.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55e4d1485f00 0x55e4d1488310 crc :-1 s=READY pgs=122 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55e4d1418d30 0x55e4d147d000 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55e4d140e4f0 0x55e4d147e2b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 --2- 172.21.15.40:0/2399811173 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55e4d140ee00 0x55e4d147e7f0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 >> 172.21.15.40:0/2399811173 conn(0x55e4d1403ff0 msgr2=0x55e4d140d440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.449+0000 7efc16ff5700 0 client.26597 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:06.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 shutdown_connections 2023-07-14T11:25:06.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc4cf9fb80 1 -- 172.21.15.40:0/2399811173 wait complete. 2023-07-14T11:25:06.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc4cf9fb80 20 client.26597 trim_cache size 0 max 16384 2023-07-14T11:25:06.466 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:25:06.467 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:25:06.467 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:25:06.467 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:25:06.467 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:25:06.467 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:25:06.468 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:25:06.468 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:25:06.468 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:25:06.468 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:25:06.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 msgr2=0x7efc1c0a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 0x7efc1c0a3890 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7efc2c009550 tx=0x7efc2c007910 comp rx=0 tx=0).stop 2023-07-14T11:25:06.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 msgr2=0x7efc3419da50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.450+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7efc2400e9a0 tx=0x7efc2400ed60 comp rx=0 tx=0).stop 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 shutdown_connections 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7efc1c0a13e0 0x7efc1c0a3890 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc3410c3b0 0x7efc34197a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7efc340fe800 0x7efc3419da50 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 --2- 172.21.15.40:0/757469544 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7efc340ff110 0x7efc3419df90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:06.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 >> 172.21.15.40:0/757469544 conn(0x7efc340745d0 msgr2=0x7efc340fc430 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:06.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 shutdown_connections 2023-07-14T11:25:06.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:06.451+0000 7efc33fff700 1 -- 172.21.15.40:0/757469544 wait complete. 2023-07-14T11:25:06.472 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:25:08.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:07 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:07 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:08.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:07 smithi130 ceph-mon[129405]: pgmap v404: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:25:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:07 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:08.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:07 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:08.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:07 smithi040 ceph-mon[97954]: pgmap v404: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:25:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:07 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:07 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:08.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:07 smithi045 ceph-mon[128791]: pgmap v404: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:25:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: 287 slow requests, 5 included below; oldest blocked for > 667.169203 secs 2023-07-14T11:25:09.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: slow request 666.713042 seconds old, received at 2023-07-14T11:14:01.389717+0000: peer_request:mds.1:2603 currently dispatched 2023-07-14T11:25:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: slow request 666.707062 seconds old, received at 2023-07-14T11:14:01.395697+0000: peer_request:mds.1:2635 currently dispatched 2023-07-14T11:25:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: slow request 666.701164 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: slow request 666.695370 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:09.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:08 smithi130 ceph-mon[129405]: slow request 666.689397 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: 287 slow requests, 5 included below; oldest blocked for > 667.169203 secs 2023-07-14T11:25:09.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: slow request 666.713042 seconds old, received at 2023-07-14T11:14:01.389717+0000: peer_request:mds.1:2603 currently dispatched 2023-07-14T11:25:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: slow request 666.707062 seconds old, received at 2023-07-14T11:14:01.395697+0000: peer_request:mds.1:2635 currently dispatched 2023-07-14T11:25:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: slow request 666.701164 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: slow request 666.695370 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:09.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:08 smithi040 ceph-mon[97954]: slow request 666.689397 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:09.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: 287 slow requests, 5 included below; oldest blocked for > 667.169203 secs 2023-07-14T11:25:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: slow request 666.713042 seconds old, received at 2023-07-14T11:14:01.389717+0000: peer_request:mds.1:2603 currently dispatched 2023-07-14T11:25:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: slow request 666.707062 seconds old, received at 2023-07-14T11:14:01.395697+0000: peer_request:mds.1:2635 currently dispatched 2023-07-14T11:25:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: slow request 666.701164 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: slow request 666.695370 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:09.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:08 smithi045 ceph-mon[128791]: slow request 666.689397 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:09 smithi130 ceph-mon[129405]: osdmap e106: 12 total, 12 up, 12 in 2023-07-14T11:25:10.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:09 smithi130 ceph-mon[129405]: pgmap v406: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:09 smithi040 ceph-mon[97954]: osdmap e106: 12 total, 12 up, 12 in 2023-07-14T11:25:10.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:09 smithi040 ceph-mon[97954]: pgmap v406: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:09 smithi045 ceph-mon[128791]: osdmap e106: 12 total, 12 up, 12 in 2023-07-14T11:25:10.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:09 smithi045 ceph-mon[128791]: pgmap v406: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:11.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: 258 slow requests, 5 included below; oldest blocked for > 672.434661 secs 2023-07-14T11:25:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: slow request 670.489048 seconds old, received at 2023-07-14T11:14:00.048192+0000: client_request(mds.1:2650 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: slow request 670.462688 seconds old, received at 2023-07-14T11:14:00.074552+0000: client_request(mds.1:2682 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: slow request 670.435892 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: slow request 670.409784 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:10 smithi130 ceph-mon[129405]: slow request 670.382989 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: 258 slow requests, 5 included below; oldest blocked for > 672.434661 secs 2023-07-14T11:25:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: slow request 670.489048 seconds old, received at 2023-07-14T11:14:00.048192+0000: client_request(mds.1:2650 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: slow request 670.462688 seconds old, received at 2023-07-14T11:14:00.074552+0000: client_request(mds.1:2682 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: slow request 670.435892 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: slow request 670.409784 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:10 smithi040 ceph-mon[97954]: slow request 670.382989 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: 258 slow requests, 5 included below; oldest blocked for > 672.434661 secs 2023-07-14T11:25:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: slow request 670.489048 seconds old, received at 2023-07-14T11:14:00.048192+0000: client_request(mds.1:2650 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: slow request 670.462688 seconds old, received at 2023-07-14T11:14:00.074552+0000: client_request(mds.1:2682 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: slow request 670.435892 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: slow request 670.409784 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:11.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:10 smithi045 ceph-mon[128791]: slow request 670.382989 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:12.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:11 smithi130 ceph-mon[129405]: pgmap v407: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:12.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:11 smithi040 ceph-mon[97954]: pgmap v407: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:12.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:11 smithi045 ceph-mon[128791]: pgmap v407: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:12 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:13.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:12 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:12 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:13.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:12 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:12 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:13.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:12 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: pgmap v408: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: 216 slow requests, 5 included below; oldest blocked for > 672.169305 secs 2023-07-14T11:25:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: slow request 671.701266 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: slow request 671.695472 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:14.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: slow request 671.689499 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: slow request 671.683648 seconds old, received at 2023-07-14T11:14:01.419213+0000: peer_request:mds.1:2763 currently dispatched 2023-07-14T11:25:14.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:13 smithi130 ceph-mon[129405]: slow request 671.677780 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: pgmap v408: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:14.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: 216 slow requests, 5 included below; oldest blocked for > 672.169305 secs 2023-07-14T11:25:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: slow request 671.701266 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: slow request 671.695472 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: slow request 671.689499 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: slow request 671.683648 seconds old, received at 2023-07-14T11:14:01.419213+0000: peer_request:mds.1:2763 currently dispatched 2023-07-14T11:25:14.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:13 smithi040 ceph-mon[97954]: slow request 671.677780 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:14.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: pgmap v408: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:25:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: 216 slow requests, 5 included below; oldest blocked for > 672.169305 secs 2023-07-14T11:25:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: slow request 671.701266 seconds old, received at 2023-07-14T11:14:01.401595+0000: peer_request:mds.1:2667 currently dispatched 2023-07-14T11:25:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: slow request 671.695472 seconds old, received at 2023-07-14T11:14:01.407389+0000: peer_request:mds.1:2699 currently dispatched 2023-07-14T11:25:14.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: slow request 671.689499 seconds old, received at 2023-07-14T11:14:01.413362+0000: peer_request:mds.1:2731 currently dispatched 2023-07-14T11:25:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: slow request 671.683648 seconds old, received at 2023-07-14T11:14:01.419213+0000: peer_request:mds.1:2763 currently dispatched 2023-07-14T11:25:14.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:13 smithi045 ceph-mon[128791]: slow request 671.677780 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:15.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:15 smithi045 ceph-mon[128791]: osdmap e107: 12 total, 12 up, 12 in 2023-07-14T11:25:15.608 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:15 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:15.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:15 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:15.609 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:15 smithi045 ceph-mon[128791]: pgmap v410: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:25:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:15 smithi130 ceph-mon[129405]: osdmap e107: 12 total, 12 up, 12 in 2023-07-14T11:25:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:15 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:15 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:15 smithi130 ceph-mon[129405]: pgmap v410: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:25:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:15 smithi040 ceph-mon[97954]: osdmap e107: 12 total, 12 up, 12 in 2023-07-14T11:25:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:15 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:15 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:15 smithi040 ceph-mon[97954]: pgmap v410: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:25:17.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: 173 slow requests, 5 included below; oldest blocked for > 677.447904 secs 2023-07-14T11:25:17.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: slow request 675.449135 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: slow request 675.423027 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: slow request 675.396232 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: slow request 675.370550 seconds old, received at 2023-07-14T11:14:00.179934+0000: client_request(mds.1:2810 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:16 smithi130 ceph-mon[129405]: slow request 675.344325 seconds old, received at 2023-07-14T11:14:00.206158+0000: client_request(mds.1:2842 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: 173 slow requests, 5 included below; oldest blocked for > 677.447904 secs 2023-07-14T11:25:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: slow request 675.449135 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: slow request 675.423027 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: slow request 675.396232 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: slow request 675.370550 seconds old, received at 2023-07-14T11:14:00.179934+0000: client_request(mds.1:2810 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:16 smithi040 ceph-mon[97954]: slow request 675.344325 seconds old, received at 2023-07-14T11:14:00.206158+0000: client_request(mds.1:2842 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: 173 slow requests, 5 included below; oldest blocked for > 677.447904 secs 2023-07-14T11:25:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: slow request 675.449135 seconds old, received at 2023-07-14T11:14:00.101348+0000: client_request(mds.1:2714 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: slow request 675.423027 seconds old, received at 2023-07-14T11:14:00.127456+0000: client_request(mds.1:2746 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: slow request 675.396232 seconds old, received at 2023-07-14T11:14:00.154251+0000: client_request(mds.1:2778 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: slow request 675.370550 seconds old, received at 2023-07-14T11:14:00.179934+0000: client_request(mds.1:2810 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:17.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:16 smithi045 ceph-mon[128791]: slow request 675.344325 seconds old, received at 2023-07-14T11:14:00.206158+0000: client_request(mds.1:2842 rename #0x10000000002/0000000100000000000000E1 #0x60b/100000005f2 caller_uid=0, caller_gid=0{}) currently failed to xlock, waiting 2023-07-14T11:25:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:17 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:18.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:17 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:18.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:17 smithi130 ceph-mon[129405]: pgmap v411: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:25:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:17 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:17 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:18.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:17 smithi040 ceph-mon[97954]: pgmap v411: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:25:18.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:17 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:17 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:18.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:17 smithi045 ceph-mon[128791]: pgmap v411: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 254 B/s wr, 12 op/s 2023-07-14T11:25:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: 96 slow requests, 5 included below; oldest blocked for > 677.169456 secs 2023-07-14T11:25:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: slow request 676.677930 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: slow request 676.672039 seconds old, received at 2023-07-14T11:14:01.430973+0000: peer_request:mds.1:2827 currently dispatched 2023-07-14T11:25:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: slow request 676.666157 seconds old, received at 2023-07-14T11:14:01.436855+0000: peer_request:mds.1:2859 currently dispatched 2023-07-14T11:25:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: slow request 676.677761 seconds old, received at 2023-07-14T11:14:01.425251+0000: peer_request:mds.1:2796 currently dispatched 2023-07-14T11:25:19.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:18 smithi130 ceph-mon[129405]: slow request 676.671844 seconds old, received at 2023-07-14T11:14:01.431168+0000: peer_request:mds.1:2828 currently dispatched 2023-07-14T11:25:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: 96 slow requests, 5 included below; oldest blocked for > 677.169456 secs 2023-07-14T11:25:19.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: slow request 676.677930 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: slow request 676.672039 seconds old, received at 2023-07-14T11:14:01.430973+0000: peer_request:mds.1:2827 currently dispatched 2023-07-14T11:25:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: slow request 676.666157 seconds old, received at 2023-07-14T11:14:01.436855+0000: peer_request:mds.1:2859 currently dispatched 2023-07-14T11:25:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: slow request 676.677761 seconds old, received at 2023-07-14T11:14:01.425251+0000: peer_request:mds.1:2796 currently dispatched 2023-07-14T11:25:19.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:18 smithi040 ceph-mon[97954]: slow request 676.671844 seconds old, received at 2023-07-14T11:14:01.431168+0000: peer_request:mds.1:2828 currently dispatched 2023-07-14T11:25:19.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: 96 slow requests, 5 included below; oldest blocked for > 677.169456 secs 2023-07-14T11:25:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: slow request 676.677930 seconds old, received at 2023-07-14T11:14:01.425081+0000: peer_request:mds.1:2795 currently dispatched 2023-07-14T11:25:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: slow request 676.672039 seconds old, received at 2023-07-14T11:14:01.430973+0000: peer_request:mds.1:2827 currently dispatched 2023-07-14T11:25:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: slow request 676.666157 seconds old, received at 2023-07-14T11:14:01.436855+0000: peer_request:mds.1:2859 currently dispatched 2023-07-14T11:25:19.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: slow request 676.677761 seconds old, received at 2023-07-14T11:14:01.425251+0000: peer_request:mds.1:2796 currently dispatched 2023-07-14T11:25:19.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:18 smithi045 ceph-mon[128791]: slow request 676.671844 seconds old, received at 2023-07-14T11:14:01.431168+0000: peer_request:mds.1:2828 currently dispatched 2023-07-14T11:25:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:19 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:20.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:19 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:20.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:19 smithi130 ceph-mon[129405]: pgmap v412: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:25:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:19 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:19 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:20.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:19 smithi040 ceph-mon[97954]: pgmap v412: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:25:20.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:19 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:19 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:20.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:19 smithi045 ceph-mon[128791]: pgmap v412: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 204 B/s wr, 10 op/s 2023-07-14T11:25:22.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:21 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:21 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:22.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:21 smithi130 ceph-mon[129405]: pgmap v413: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:22.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:21 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:22.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:21 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:22.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:21 smithi040 ceph-mon[97954]: pgmap v413: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:22.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:21 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:21 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:22.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:21 smithi045 ceph-mon[128791]: pgmap v413: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:23.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:22 smithi130 ceph-mon[129405]: MDS health message cleared (mds.?): 173 slow requests are blocked > 180 secs 2023-07-14T11:25:23.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:22 smithi040 ceph-mon[97954]: MDS health message cleared (mds.?): 173 slow requests are blocked > 180 secs 2023-07-14T11:25:23.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:22 smithi045 ceph-mon[128791]: MDS health message cleared (mds.?): 173 slow requests are blocked > 180 secs 2023-07-14T11:25:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:23 smithi130 ceph-mon[129405]: Health check update: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:25:24.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:23 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:23 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:24.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:23 smithi130 ceph-mon[129405]: pgmap v414: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:23 smithi040 ceph-mon[97954]: Health check update: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:25:24.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:23 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:23 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:24.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:23 smithi040 ceph-mon[97954]: pgmap v414: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:24.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:23 smithi045 ceph-mon[128791]: Health check update: 1 MDSs report slow requests (MDS_SLOW_REQUEST) 2023-07-14T11:25:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:23 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6840/1989097261,v1:172.21.15.40:6841/1989097261] up:active 2023-07-14T11:25:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:23 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:24.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:23 smithi045 ceph-mon[128791]: pgmap v414: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:25:25.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:24 smithi130 ceph-mon[129405]: MDS health message cleared (mds.?): 96 slow requests are blocked > 180 secs 2023-07-14T11:25:25.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:24 smithi040 ceph-mon[97954]: MDS health message cleared (mds.?): 96 slow requests are blocked > 180 secs 2023-07-14T11:25:25.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:24 smithi045 ceph-mon[128791]: MDS health message cleared (mds.?): 96 slow requests are blocked > 180 secs 2023-07-14T11:25:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:25 smithi130 ceph-mon[129405]: Health check cleared: MDS_SLOW_REQUEST (was: 1 MDSs report slow requests) 2023-07-14T11:25:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:25 smithi130 ceph-mon[129405]: Cluster is now healthy 2023-07-14T11:25:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:25 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:26.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:25 smithi130 ceph-mon[129405]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:26.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:25 smithi130 ceph-mon[129405]: pgmap v415: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:25:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:25 smithi040 ceph-mon[97954]: Health check cleared: MDS_SLOW_REQUEST (was: 1 MDSs report slow requests) 2023-07-14T11:25:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:25 smithi040 ceph-mon[97954]: Cluster is now healthy 2023-07-14T11:25:26.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:25 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:25 smithi040 ceph-mon[97954]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:26.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:25 smithi040 ceph-mon[97954]: pgmap v415: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:25:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:25 smithi045 ceph-mon[128791]: Health check cleared: MDS_SLOW_REQUEST (was: 1 MDSs report slow requests) 2023-07-14T11:25:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:25 smithi045 ceph-mon[128791]: Cluster is now healthy 2023-07-14T11:25:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:25 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] up:active 2023-07-14T11:25:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:25 smithi045 ceph-mon[128791]: fsmap cephfs:5 5 up:standby-replay 2 up:standby 5 up:active 2023-07-14T11:25:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:25 smithi045 ceph-mon[128791]: pgmap v415: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 7.5 KiB/s rd, 100 B/s wr, 14 op/s 2023-07-14T11:25:28.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:27 smithi045 ceph-mon[128791]: pgmap v416: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:27 smithi130 ceph-mon[129405]: pgmap v416: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:27 smithi040 ceph-mon[97954]: pgmap v416: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:30.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:29 smithi045 ceph-mon[128791]: pgmap v417: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:29 smithi130 ceph-mon[129405]: pgmap v417: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:29 smithi040 ceph-mon[97954]: pgmap v417: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:32.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:31 smithi045 ceph-mon[128791]: pgmap v418: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:32.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:31 smithi130 ceph-mon[129405]: pgmap v418: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:31 smithi040 ceph-mon[97954]: pgmap v418: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:34.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:33 smithi045 ceph-mon[128791]: pgmap v419: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:33 smithi130 ceph-mon[129405]: pgmap v419: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:33 smithi040 ceph-mon[97954]: pgmap v419: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:35 smithi130 ceph-mon[129405]: pgmap v420: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:35 smithi040 ceph-mon[97954]: pgmap v420: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:35 smithi045 ceph-mon[128791]: pgmap v420: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:36.473 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:25:36.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 -- 172.21.15.40:0/1419264694 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7218100450 msgr2=0x7f721810c8e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 --2- 172.21.15.40:0/1419264694 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7218100450 0x7f721810c8e0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f721000b0a0 tx=0x7f721000b3b0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 -- 172.21.15.40:0/1419264694 shutdown_connections 2023-07-14T11:25:36.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 --2- 172.21.15.40:0/1419264694 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f7218100450 0x7f721810c8e0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 --2- 172.21.15.40:0/1419264694 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218105ab0 0x7f7218105e80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 --2- 172.21.15.40:0/1419264694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f72180ffaa0 0x7f72180fff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.692 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 -- 172.21.15.40:0/1419264694 >> 172.21.15.40:0/1419264694 conn(0x7f72180fb5a0 msgr2=0x7f72180fd9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 -- 172.21.15.40:0/1419264694 shutdown_connections 2023-07-14T11:25:36.693 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.688+0000 7f721d9ff700 1 -- 172.21.15.40:0/1419264694 wait complete. 2023-07-14T11:25:36.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.689+0000 7f721d9ff700 1 Processor -- start 2023-07-14T11:25:36.694 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 -- start start 2023-07-14T11:25:36.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 0x7f7218198390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7218105ab0 0x7f721819c7f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721819ce30 con 0x7f7218105ab0 2023-07-14T11:25:36.695 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721819cfa0 con 0x7f72180ffaa0 2023-07-14T11:25:36.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f721d9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721819d160 con 0x7f7218100450 2023-07-14T11:25:36.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41768/0 (socket says 172.21.15.40:41768) 2023-07-14T11:25:36.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f7216ffd700 1 -- 172.21.15.40:0/2315517165 learned_addr learned my addr 172.21.15.40:0/2315517165 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:36.696 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f72167fc700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 0x7f7218198390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.690+0000 7f72177fe700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7218105ab0 0x7f721819c7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 -- 172.21.15.40:0/2315517165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 msgr2=0x7f7218198390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 0x7f7218198390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 -- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7218105ab0 msgr2=0x7f721819c7f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.697 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7218105ab0 0x7f721819c7f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 -- 172.21.15.40:0/2315517165 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7208009520 con 0x7f72180ffaa0 2023-07-14T11:25:36.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f72167fc700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 0x7f7218198390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:36.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f7216ffd700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f720800ea50 tx=0x7f720800ed60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f721c9fd700 1 -- 172.21.15.40:0/2315517165 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f720800cbd0 con 0x7f72180ffaa0 2023-07-14T11:25:36.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f721c9fd700 1 -- 172.21.15.40:0/2315517165 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f720800cd30 con 0x7f72180ffaa0 2023-07-14T11:25:36.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7210009d00 con 0x7f72180ffaa0 2023-07-14T11:25:36.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f721c9fd700 1 -- 172.21.15.40:0/2315517165 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7208018680 con 0x7f72180ffaa0 2023-07-14T11:25:36.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.691+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f721819d630 con 0x7f72180ffaa0 2023-07-14T11:25:36.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.695+0000 7f721c9fd700 1 -- 172.21.15.40:0/2315517165 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f7208010430 con 0x7f72180ffaa0 2023-07-14T11:25:36.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.695+0000 7f7231223b80 1 Processor -- start 2023-07-14T11:25:36.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 -- start start 2023-07-14T11:25:36.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 0x562b02f7a6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f85ad0 0x562b02f87eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02e80cb0 con 0x562b02f85ad0 2023-07-14T11:25:36.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02ed3a40 con 0x562b02f7abe0 2023-07-14T11:25:36.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7231223b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02ecced0 con 0x562b02f7a2d0 2023-07-14T11:25:36.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 0x562b02f7a6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 0x562b02f7a6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47532/0 (socket says 172.21.15.40:47532) 2023-07-14T11:25:36.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f72177fe700 1 -- 172.21.15.40:0/1202391807 learned_addr learned my addr 172.21.15.40:0/1202391807 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:36.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41780/0 (socket says 172.21.15.40:41780) 2023-07-14T11:25:36.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.696+0000 7f72167fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f85ad0 0x562b02f87eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f721c9fd700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 0x7f72000a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 -- 172.21.15.40:0/1202391807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 msgr2=0x562b02f7a6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 0x562b02f7a6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 -- 172.21.15.40:0/1202391807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f85ad0 msgr2=0x562b02f87eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f85ad0 0x562b02f87eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 -- 172.21.15.40:0/1202391807 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562b02cfade0 con 0x562b02f7abe0 2023-07-14T11:25:36.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f721c9fd700 1 -- 172.21.15.40:0/2315517165 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f7208014070 con 0x7f72180ffaa0 2023-07-14T11:25:36.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f72177fe700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 0x7f72000a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f7216ffd700 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f72080ca870 tx=0x7f720800ef80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b5da7105d8fd8576 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f71fdffb700 1 -- 172.21.15.40:0/1202391807 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7208099d30 con 0x562b02f7abe0 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f71fdffb700 1 -- 172.21.15.40:0/1202391807 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7208099b50 con 0x562b02f7abe0 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.697+0000 7f71fdffb700 1 -- 172.21.15.40:0/1202391807 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7208098330 con 0x562b02f7abe0 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 -- 172.21.15.40:0/1202391807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 msgr2=0x562b02f85590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f72177fe700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 0x7f72000a37f0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f721000b0a0 tx=0x7f72100099e0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f72080ca870 tx=0x7f720800ef80 comp rx=0 tx=0).stop 2023-07-14T11:25:36.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 -- 172.21.15.40:0/1202391807 shutdown_connections 2023-07-14T11:25:36.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f85ad0 0x562b02f87eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f7a2d0 0x562b02f7a6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 --2- 172.21.15.40:0/1202391807 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f85590 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 -- 172.21.15.40:0/1202391807 >> 172.21.15.40:0/1202391807 conn(0x562b02f70da0 msgr2=0x562b02f766a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 -- 172.21.15.40:0/1202391807 shutdown_connections 2023-07-14T11:25:36.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.698+0000 7f7231223b80 1 -- 172.21.15.40:0/1202391807 wait complete. 2023-07-14T11:25:36.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 -- start start 2023-07-14T11:25:36.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 Processor -- start 2023-07-14T11:25:36.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f827a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82ee0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02e6cac0 con 0x562b02f7abe0 2023-07-14T11:25:36.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02e6c7c0 con 0x562b02f7a2d0 2023-07-14T11:25:36.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.699+0000 7f7231223b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02d50920 con 0x562b02f85ad0 2023-07-14T11:25:36.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7231223b80 10 client.0 _ll_register_callbacks cb 0x562b02e6a760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:25:36.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41782/0 (socket says 172.21.15.40:41782) 2023-07-14T11:25:36.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 -- 172.21.15.40:0/856466688 learned_addr learned my addr 172.21.15.40:0/856466688 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:36.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f72167fc700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f827a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f72177fe700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 -- 172.21.15.40:0/856466688 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 msgr2=0x562b02f82ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 -- 172.21.15.40:0/856466688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 msgr2=0x562b02f827a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f827a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 -- 172.21.15.40:0/856466688 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f721819d630 con 0x562b02f7a2d0 2023-07-14T11:25:36.714 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f7216ffd700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f720800c5b0 tx=0x7f72080d0eb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=fcf8599435a4b80c server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f71f6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.700+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7208099d30 con 0x562b02f7a2d0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f720809a1f0 con 0x562b02f7a2d0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x562b02e264f0 con 0x562b02f7a2d0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7208093140 con 0x562b02f7a2d0 2023-07-14T11:25:36.715 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 10 client.26609 fetch_fsmap learned FSMap version 252 2023-07-14T11:25:36.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x562b02ce3c60 con 0x562b02f7a2d0 2023-07-14T11:25:36.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x562b02e264f0 con 0x562b02f7a2d0 2023-07-14T11:25:36.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f720809a450 con 0x562b02f7a2d0 2023-07-14T11:25:36.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 10 client.26609 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:25:36.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 10 client.26609 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:25:36.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 20 client.26609 populate_metadata read hostname 'smithi040' 2023-07-14T11:25:36.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02f94f30 0x562b02f97310 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 4 client.26609 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:25:36.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.701+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562b02ec5eb0 con 0x562b02f94f30 2023-07-14T11:25:36.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.702+0000 7f72167fc700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02f94f30 0x562b02f97310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.702+0000 7f72167fc700 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02f94f30 0x562b02f97310 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=57ee3611d08837ff server_cookie=afb9064bf9b77768 in_seq=0 out_seq=0 2023-07-14T11:25:36.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.702+0000 7f71f6ffd700 10 client.26609 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:36.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f71f6ffd700 1 -- 172.21.15.40:0/856466688 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562b02ec5eb0 con 0x562b02f94f30 2023-07-14T11:25:36.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f71f6ffd700 10 client.26609 handle_command_reply: tid=0 2023-07-14T11:25:36.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 client.26609 shutdown 2023-07-14T11:25:36.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 msgr2=0x562b02f85590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f720800c5b0 tx=0x7f72080d0eb0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 shutdown_connections 2023-07-14T11:25:36.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02f94f30 0x562b02f97310 crc :-1 s=READY pgs=123 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f85590 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f71f6ffd700 0 client.26609 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:36.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 --2- 172.21.15.40:0/856466688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f827a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 >> 172.21.15.40:0/856466688 conn(0x562b02f70da0 msgr2=0x562b02f76230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 shutdown_connections 2023-07-14T11:25:36.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.703+0000 7f7231223b80 1 -- 172.21.15.40:0/856466688 wait complete. 2023-07-14T11:25:36.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.704+0000 7f7231223b80 20 client.26609 trim_cache size 0 max 16384 2023-07-14T11:25:36.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 Processor -- start 2023-07-14T11:25:36.723 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 -- start start 2023-07-14T11:25:36.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f82040 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.725 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02f91360 con 0x562b02f7a2d0 2023-07-14T11:25:36.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02f77d20 con 0x562b02f7abe0 2023-07-14T11:25:36.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.721+0000 7f7231223b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02f7d690 con 0x562b02f85ad0 2023-07-14T11:25:36.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.726 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f72167fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f82040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:56226/0 (socket says 172.21.15.40:56226) 2023-07-14T11:25:36.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47542/0 (socket says 172.21.15.40:47542) 2023-07-14T11:25:36.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 -- 172.21.15.40:0/910306422 learned_addr learned my addr 172.21.15.40:0/910306422 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:36.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 -- 172.21.15.40:0/910306422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 msgr2=0x562b02f82580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.727 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 -- 172.21.15.40:0/910306422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 msgr2=0x562b02f82040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f82040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 -- 172.21.15.40:0/910306422 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562b02d21250 con 0x562b02f7a2d0 2023-07-14T11:25:36.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f72177fe700 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:25:36.728 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f7216ffd700 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f7208098540 tx=0x7f720800eef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=475303ee19033c8e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.722+0000 7f71fe7fc700 1 -- 172.21.15.40:0/910306422 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f7208099d30 con 0x562b02f7a2d0 2023-07-14T11:25:36.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f71fe7fc700 1 -- 172.21.15.40:0/910306422 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7208095fe0 con 0x562b02f7a2d0 2023-07-14T11:25:36.729 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f71fe7fc700 1 -- 172.21.15.40:0/910306422 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f720809e2b0 con 0x562b02f7a2d0 2023-07-14T11:25:36.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 -- 172.21.15.40:0/910306422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 msgr2=0x562b02f850c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 crypto rx=0x7f7208098540 tx=0x7f720800eef0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 -- 172.21.15.40:0/910306422 shutdown_connections 2023-07-14T11:25:36.732 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02f82580 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7a2d0 0x562b02f850c0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 --2- 172.21.15.40:0/910306422 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7abe0 0x562b02f82040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 -- 172.21.15.40:0/910306422 >> 172.21.15.40:0/910306422 conn(0x562b02f70da0 msgr2=0x562b02f90c50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 -- 172.21.15.40:0/910306422 shutdown_connections 2023-07-14T11:25:36.733 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.723+0000 7f7231223b80 1 -- 172.21.15.40:0/910306422 wait complete. 2023-07-14T11:25:36.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 -- start start 2023-07-14T11:25:36.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 Processor -- start 2023-07-14T11:25:36.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f8a680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f8abc0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.734 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02fe23b0 con 0x562b02f7abe0 2023-07-14T11:25:36.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02fe2520 con 0x562b02f7a2d0 2023-07-14T11:25:36.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x562b02fe26e0 con 0x562b02f85ad0 2023-07-14T11:25:36.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.724+0000 7f7231223b80 10 client.0 _ll_register_callbacks cb 0x7f720c009be0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:25:36.735 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:47554/0 (socket says 172.21.15.40:47554) 2023-07-14T11:25:36.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 -- 172.21.15.40:0/3678732635 learned_addr learned my addr 172.21.15.40:0/3678732635 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:25:36.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f7216ffd700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f8a680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72167fc700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f8abc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.736 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 -- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 msgr2=0x562b02f8a680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f8a680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 -- 172.21.15.40:0/3678732635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 msgr2=0x562b02f8abc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:25:36.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f8abc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 -- 172.21.15.40:0/3678732635 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f721810d1e0 con 0x562b02f85ad0 2023-07-14T11:25:36.737 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f72177fe700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f7210007760 tx=0x7f7210006230 comp rx=0 tx=0).ready entity=mon.1 client_cookie=f45fd03f2b463c03 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:25:36.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f71f6ffd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:25:36.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.725+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f721000e040 con 0x562b02f85ad0 2023-07-14T11:25:36.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f7208099b50 con 0x562b02f85ad0 2023-07-14T11:25:36.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f7210005450 con 0x562b02f85ad0 2023-07-14T11:25:36.738 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f7210005650 con 0x562b02f85ad0 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 10 client.26578 fetch_fsmap learned FSMap version 252 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x562b02ce3c60 con 0x562b02f85ad0 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f7208099b50 con 0x562b02f85ad0 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f721001e9d0 con 0x562b02f85ad0 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 10 client.26578 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:25:36.739 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 10 client.26578 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:25:36.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 20 client.26578 populate_metadata read hostname 'smithi040' 2023-07-14T11:25:36.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02fe9e20 0x562b02fec230 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:25:36.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 4 client.26578 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:25:36.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.726+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x562b02e88050 con 0x562b02fe9e20 2023-07-14T11:25:36.740 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f7216ffd700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02fe9e20 0x562b02fec230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:25:36.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f7216ffd700 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02fe9e20 0x562b02fec230 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c59cbed5d4f9c567 server_cookie=d266f0b5fef21eeb in_seq=0 out_seq=0 2023-07-14T11:25:36.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f71f6ffd700 10 client.26578 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:36.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f71f6ffd700 1 -- 172.21.15.40:0/3678732635 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x562b02e88050 con 0x562b02fe9e20 2023-07-14T11:25:36.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f71f6ffd700 10 client.26578 handle_command_reply: tid=0 2023-07-14T11:25:36.741 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.727+0000 7f7231223b80 1 client.26578 shutdown 2023-07-14T11:25:36.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 msgr2=0x562b02fe1e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f7210007760 tx=0x7f7210006230 comp rx=0 tx=0).stop 2023-07-14T11:25:36.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 shutdown_connections 2023-07-14T11:25:36.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x562b02fe9e20 0x562b02fec230 crc :-1 s=READY pgs=124 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.742 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x562b02f85ad0 0x562b02fe1e70 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x562b02f7a2d0 0x562b02f8a680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 --2- 172.21.15.40:0/3678732635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x562b02f7abe0 0x562b02f8abc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f71f6ffd700 0 client.26578 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:25:36.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 >> 172.21.15.40:0/3678732635 conn(0x562b02f70da0 msgr2=0x562b02f82040 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.743 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 shutdown_connections 2023-07-14T11:25:36.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 1 -- 172.21.15.40:0/3678732635 wait complete. 2023-07-14T11:25:36.744 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f7231223b80 20 client.26578 trim_cache size 0 max 16384 2023-07-14T11:25:36.744 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:25:36.745 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:25:36.746 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:25:36.746 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:25:36.746 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:25:36.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.728+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 msgr2=0x7f72000a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 0x7f72000a37f0 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7f721000b0a0 tx=0x7f72100099e0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 msgr2=0x7f7218197e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:25:36.747 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f720800ea50 tx=0x7f720800ed60 comp rx=0 tx=0).stop 2023-07-14T11:25:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 shutdown_connections 2023-07-14T11:25:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f72000a1340 0x7f72000a37f0 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7218105ab0 0x7f721819c7f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f72180ffaa0 0x7f7218197e50 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.748 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 --2- 172.21.15.40:0/2315517165 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f7218100450 0x7f7218198390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:25:36.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 >> 172.21.15.40:0/2315517165 conn(0x7f72180fb5a0 msgr2=0x7f7218072740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:25:36.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 shutdown_connections 2023-07-14T11:25:36.749 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:25:36.729+0000 7f721d9ff700 1 -- 172.21.15.40:0/2315517165 wait complete. 2023-07-14T11:25:36.750 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:25:38.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:37 smithi045 ceph-mon[128791]: pgmap v421: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:37 smithi130 ceph-mon[129405]: pgmap v421: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:38.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:37 smithi040 ceph-mon[97954]: pgmap v421: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:40.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:39 smithi045 ceph-mon[128791]: pgmap v422: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:39 smithi130 ceph-mon[129405]: pgmap v422: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:39 smithi040 ceph-mon[97954]: pgmap v422: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:42.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:41 smithi045 ceph-mon[128791]: pgmap v423: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:42.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:41 smithi130 ceph-mon[129405]: pgmap v423: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:42.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:41 smithi040 ceph-mon[97954]: pgmap v423: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:44.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:43 smithi045 ceph-mon[128791]: pgmap v424: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:44.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:43 smithi130 ceph-mon[129405]: pgmap v424: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:43 smithi040 ceph-mon[97954]: pgmap v424: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:45 smithi130 ceph-mon[129405]: pgmap v425: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:45 smithi040 ceph-mon[97954]: pgmap v425: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:45 smithi045 ceph-mon[128791]: pgmap v425: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:48.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:47 smithi045 ceph-mon[128791]: pgmap v426: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:48.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:47 smithi130 ceph-mon[129405]: pgmap v426: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:48.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:47 smithi040 ceph-mon[97954]: pgmap v426: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:50.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:49 smithi045 ceph-mon[128791]: pgmap v427: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:50.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:49 smithi130 ceph-mon[129405]: pgmap v427: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:49 smithi040 ceph-mon[97954]: pgmap v427: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:52.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:51 smithi045 ceph-mon[128791]: pgmap v428: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:51 smithi130 ceph-mon[129405]: pgmap v428: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:51 smithi040 ceph-mon[97954]: pgmap v428: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:54.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:53 smithi130 ceph-mon[129405]: pgmap v429: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:53 smithi040 ceph-mon[97954]: pgmap v429: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:54.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:53 smithi045 ceph-mon[128791]: pgmap v429: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:25:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:55 smithi130 ceph-mon[129405]: pgmap v430: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:55 smithi040 ceph-mon[97954]: pgmap v430: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:55 smithi045 ceph-mon[128791]: pgmap v430: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:25:58.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:57 smithi045 ceph-mon[128791]: pgmap v431: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:25:58.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:57 smithi130 ceph-mon[129405]: pgmap v431: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:25:58.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:57 smithi040 ceph-mon[97954]: pgmap v431: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:00.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:59 smithi045 ceph-mon[128791]: pgmap v432: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:00.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:25:59 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:26:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:59 smithi130 ceph-mon[129405]: pgmap v432: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:25:59 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:26:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:59 smithi040 ceph-mon[97954]: pgmap v432: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:25:59 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:26:01.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:00 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:26:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:00 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:26:01.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:00 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:26:01.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:00 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:26:01.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:00 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:26:01.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:00 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:26:01.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:00 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:26:01.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:00 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:26:01.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:00 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:26:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:01 smithi130 ceph-mon[129405]: pgmap v433: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:01 smithi040 ceph-mon[97954]: pgmap v433: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:02.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:01 smithi045 ceph-mon[128791]: pgmap v433: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:04.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:03 smithi130 ceph-mon[129405]: pgmap v434: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:03 smithi040 ceph-mon[97954]: pgmap v434: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:04.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:03 smithi045 ceph-mon[128791]: pgmap v434: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.1 KiB/s rd, 8 op/s 2023-07-14T11:26:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:05 smithi130 ceph-mon[129405]: pgmap v435: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:26:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:05 smithi040 ceph-mon[97954]: pgmap v435: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:26:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:05 smithi045 ceph-mon[128791]: pgmap v435: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:26:06.751 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:26:06.971 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 -- 172.21.15.40:0/2098216211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c0ffad0 msgr2=0x7ff89c0fff40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:06.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/2098216211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c0ffad0 0x7ff89c0fff40 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7ff884009990 tx=0x7ff884009ca0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 -- 172.21.15.40:0/2098216211 shutdown_connections 2023-07-14T11:26:06.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/2098216211 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c10c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.972 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/2098216211 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c105ae0 0x7ff89c105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/2098216211 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c0ffad0 0x7ff89c0fff40 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 -- 172.21.15.40:0/2098216211 >> 172.21.15.40:0/2098216211 conn(0x7ff89c0fb5f0 msgr2=0x7ff89c0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:06.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 -- 172.21.15.40:0/2098216211 shutdown_connections 2023-07-14T11:26:06.973 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.968+0000 7ff8a19ff700 1 -- 172.21.15.40:0/2098216211 wait complete. 2023-07-14T11:26:06.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.969+0000 7ff8a19ff700 1 Processor -- start 2023-07-14T11:26:06.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.969+0000 7ff8a19ff700 1 -- start start 2023-07-14T11:26:06.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.969+0000 7ff8a19ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.975 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff8a19ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c19bf70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff8a19ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c105ae0 0x7ff89c195aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff8a19ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff89c196110 con 0x7ff89c100480 2023-07-14T11:26:06.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff8a19ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff89c196280 con 0x7ff89c0ffad0 2023-07-14T11:26:06.976 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff8a19ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff89c196440 con 0x7ff89c105ae0 2023-07-14T11:26:06.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49494/0 (socket says 172.21.15.40:49494) 2023-07-14T11:26:06.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 -- 172.21.15.40:0/1270115464 learned_addr learned my addr 172.21.15.40:0/1270115464 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:06.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c105ae0 0x7ff89c195aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.977 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff893fff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c19bf70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 -- 172.21.15.40:0/1270115464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c105ae0 msgr2=0x7ff89c195aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c105ae0 0x7ff89c195aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 -- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 msgr2=0x7ff89c19bf70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c19bf70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 -- 172.21.15.40:0/1270115464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8840095f0 con 0x7ff89c0ffad0 2023-07-14T11:26:06.978 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff893fff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c19bf70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:26:06.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.970+0000 7ff89affd700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7ff88c00e9a0 tx=0x7ff88c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:06.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.971+0000 7ff8a09fd700 1 -- 172.21.15.40:0/1270115464 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88c009790 con 0x7ff89c0ffad0 2023-07-14T11:26:06.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.971+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff89c196720 con 0x7ff89c0ffad0 2023-07-14T11:26:06.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.971+0000 7ff8a09fd700 1 -- 172.21.15.40:0/1270115464 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff88c00cbd0 con 0x7ff89c0ffad0 2023-07-14T11:26:06.979 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.971+0000 7ff8a09fd700 1 -- 172.21.15.40:0/1270115464 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88c018680 con 0x7ff89c0ffad0 2023-07-14T11:26:06.980 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.971+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff89c1a2a20 con 0x7ff89c0ffad0 2023-07-14T11:26:06.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.976+0000 7ff8a09fd700 1 -- 172.21.15.40:0/1270115464 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7ff88c00cd40 con 0x7ff89c0ffad0 2023-07-14T11:26:06.981 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.976+0000 7ff8b52f7b80 1 Processor -- start 2023-07-14T11:26:06.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 -- start start 2023-07-14T11:26:06.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a78aad0 0x555c9a78ceb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.982 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a685cb0 con 0x555c9a77f2d0 2023-07-14T11:26:06.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a6d8a40 con 0x555c9a78aad0 2023-07-14T11:26:06.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a6d1ed0 con 0x555c9a77fbe0 2023-07-14T11:26:06.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff89affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff89b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.983 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff89affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60804/0 (socket says 172.21.15.40:60804) 2023-07-14T11:26:06.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff89affd700 1 -- 172.21.15.40:0/1022291985 learned_addr learned my addr 172.21.15.40:0/1022291985 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:06.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:59182/0 (socket says 172.21.15.40:59182) 2023-07-14T11:26:06.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.977+0000 7ff893fff700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a78aad0 0x555c9a78ceb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 -- 172.21.15.40:0/1022291985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a78aad0 msgr2=0x555c9a78ceb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:06.984 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a78aad0 0x555c9a78ceb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 -- 172.21.15.40:0/1022291985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 msgr2=0x555c9a77f6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:06.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 -- 172.21.15.40:0/1022291985 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555c9a4ffde0 con 0x555c9a77fbe0 2023-07-14T11:26:06.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff8a09fd700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 0x7ff8880a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.985 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:26:06.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff8a09fd700 1 -- 172.21.15.40:0/1270115464 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ff88c014070 con 0x7ff89c0ffad0 2023-07-14T11:26:06.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89affd700 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff88c0ca5a0 tx=0x7ff88c0d5440 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c9ad123e5c60a0ed server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:06.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 0x7ff8880a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff891ffb700 1 -- 172.21.15.40:0/1022291985 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88c099db0 con 0x555c9a77fbe0 2023-07-14T11:26:06.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff891ffb700 1 -- 172.21.15.40:0/1022291985 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff88c099b20 con 0x555c9a77fbe0 2023-07-14T11:26:06.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.978+0000 7ff891ffb700 1 -- 172.21.15.40:0/1022291985 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88c09e010 con 0x555c9a77fbe0 2023-07-14T11:26:06.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1022291985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 msgr2=0x555c9a78a590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:06.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7ff88c0ca5a0 tx=0x7ff88c0d5440 comp rx=0 tx=0).stop 2023-07-14T11:26:06.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 0x7ff8880a3770 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ff89c19c730 tx=0x7ff89400b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:06.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1022291985 shutdown_connections 2023-07-14T11:26:06.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a78aad0 0x555c9a78ceb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a77f2d0 0x555c9a77f6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1022291985 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78a590 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1022291985 >> 172.21.15.40:0/1022291985 conn(0x555c9a775da0 msgr2=0x555c9a77b6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:06.991 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1022291985 shutdown_connections 2023-07-14T11:26:06.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.979+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1022291985 wait complete. 2023-07-14T11:26:06.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 -- start start 2023-07-14T11:26:06.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 Processor -- start 2023-07-14T11:26:06.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 0x555c9a78a520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 0x555c9a787730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a787e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a671ac0 con 0x555c9a78aad0 2023-07-14T11:26:06.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a6717c0 con 0x555c9a77fbe0 2023-07-14T11:26:06.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.980+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a555920 con 0x555c9a77f2d0 2023-07-14T11:26:06.993 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff8b52f7b80 10 client.0 _ll_register_callbacks cb 0x555c9a66f760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:26:06.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff893fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 0x555c9a787730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 0x555c9a78a520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff893fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 0x555c9a787730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49510/0 (socket says 172.21.15.40:49510) 2023-07-14T11:26:06.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff893fff700 1 -- 172.21.15.40:0/2511896908 learned_addr learned my addr 172.21.15.40:0/2511896908 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:06.994 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a787e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 -- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 msgr2=0x555c9a787730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 0x555c9a787730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 -- 172.21.15.40:0/2511896908 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 msgr2=0x555c9a787e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a787e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 -- 172.21.15.40:0/2511896908 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff894011230 con 0x555c9a77f2d0 2023-07-14T11:26:06.995 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a787e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:26:06.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff89affd700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 0x555c9a78a520 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7ff88c009230 tx=0x7ff88c0cab10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=bba7261ac6d3a27b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:06.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff87affd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:26:06.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88c099940 con 0x555c9a77f2d0 2023-07-14T11:26:06.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ff88c0cd0d0 con 0x555c9a77f2d0 2023-07-14T11:26:06.996 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.981+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x555c9a62b4f0 con 0x555c9a77f2d0 2023-07-14T11:26:06.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff88c099fe0 con 0x555c9a77f2d0 2023-07-14T11:26:06.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 10 client.26590 fetch_fsmap learned FSMap version 252 2023-07-14T11:26:06.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x7ff89c1a2a20 con 0x555c9a77f2d0 2023-07-14T11:26:06.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x555c9a62b4f0 con 0x555c9a77f2d0 2023-07-14T11:26:06.997 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ff88c09a240 con 0x555c9a77f2d0 2023-07-14T11:26:06.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 10 client.26590 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:26:06.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 10 client.26590 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:26:06.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 20 client.26590 populate_metadata read hostname 'smithi040' 2023-07-14T11:26:06.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a79a030 0x555c9a79c410 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:06.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 4 client.26590 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:26:06.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x555c9a6caeb0 con 0x555c9a79a030 2023-07-14T11:26:06.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.982+0000 7ff893fff700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a79a030 0x555c9a79c410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:06.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.983+0000 7ff893fff700 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a79a030 0x555c9a79c410 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=294ad93b3ccca9b5 server_cookie=d7a7d254b727b945 in_seq=0 out_seq=0 2023-07-14T11:26:06.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.983+0000 7ff87affd700 10 client.26590 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:06.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff87affd700 1 -- 172.21.15.40:0/2511896908 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x555c9a6caeb0 con 0x555c9a79a030 2023-07-14T11:26:07.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff87affd700 10 client.26590 handle_command_reply: tid=0 2023-07-14T11:26:07.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 client.26590 shutdown 2023-07-14T11:26:07.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 msgr2=0x555c9a78a520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:07.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 0x555c9a78a520 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7ff88c009230 tx=0x7ff88c0cab10 comp rx=0 tx=0).stop 2023-07-14T11:26:07.000 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 shutdown_connections 2023-07-14T11:26:07.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a79a030 0x555c9a79c410 crc :-1 s=READY pgs=125 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a787e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77f2d0 0x555c9a78a520 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2511896908 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77fbe0 0x555c9a787730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 >> 172.21.15.40:0/2511896908 conn(0x555c9a775da0 msgr2=0x555c9a77b230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:07.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff87affd700 0 client.26590 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:07.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 shutdown_connections 2023-07-14T11:26:07.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2511896908 wait complete. 2023-07-14T11:26:07.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:06.984+0000 7ff8b52f7b80 20 client.26590 trim_cache size 0 max 16384 2023-07-14T11:26:07.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 Processor -- start 2023-07-14T11:26:07.005 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 -- start start 2023-07-14T11:26:07.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a7869c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78a050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.007 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a780540 con 0x555c9a78aad0 2023-07-14T11:26:07.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a77ffb0 con 0x555c9a77f2d0 2023-07-14T11:26:07.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.002+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a788910 con 0x555c9a77fbe0 2023-07-14T11:26:07.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff89affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a7869c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.008 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:60822/0 (socket says 172.21.15.40:60822) 2023-07-14T11:26:07.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 -- 172.21.15.40:0/1957588601 learned_addr learned my addr 172.21.15.40:0/1957588601 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:07.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78a050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 -- 172.21.15.40:0/1957588601 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 msgr2=0x555c9a7869c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:07.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a7869c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.009 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 -- 172.21.15.40:0/1957588601 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 msgr2=0x555c9a78a050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:07.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78a050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 -- 172.21.15.40:0/1957588601 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x555c9a4e8c60 con 0x555c9a77fbe0 2023-07-14T11:26:07.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff89affd700 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a7869c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:26:07.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.003+0000 7ff893fff700 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7ff8840098e0 tx=0x7ff884009310 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6fe96477c7e22dba server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:07.010 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8927fc700 1 -- 172.21.15.40:0/1957588601 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff884029b40 con 0x555c9a77fbe0 2023-07-14T11:26:07.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8927fc700 1 -- 172.21.15.40:0/1957588601 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff88401e040 con 0x555c9a77fbe0 2023-07-14T11:26:07.011 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8927fc700 1 -- 172.21.15.40:0/1957588601 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff88400f460 con 0x555c9a77fbe0 2023-07-14T11:26:07.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1957588601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 msgr2=0x555c9a796760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:07.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7ff8840098e0 tx=0x7ff884009310 comp rx=0 tx=0).stop 2023-07-14T11:26:07.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1957588601 shutdown_connections 2023-07-14T11:26:07.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78a050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.014 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a7869c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/1957588601 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a796760 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1957588601 >> 172.21.15.40:0/1957588601 conn(0x555c9a775da0 msgr2=0x555c9a779e10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:07.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1957588601 shutdown_connections 2023-07-14T11:26:07.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.004+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/1957588601 wait complete. 2023-07-14T11:26:07.015 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 -- start start 2023-07-14T11:26:07.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 Processor -- start 2023-07-14T11:26:07.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78f940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78fe80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.016 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a790590 con 0x555c9a78aad0 2023-07-14T11:26:07.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a790700 con 0x555c9a77f2d0 2023-07-14T11:26:07.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x555c9a7ede90 con 0x555c9a77fbe0 2023-07-14T11:26:07.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.005+0000 7ff8b52f7b80 10 client.0 _ll_register_callbacks cb 0x7ff894009f30 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:26:07.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff89affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.017 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff89affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:49520/0 (socket says 172.21.15.40:49520) 2023-07-14T11:26:07.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78f940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff89affd700 1 -- 172.21.15.40:0/2255496529 learned_addr learned my addr 172.21.15.40:0/2255496529 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:07.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff89b7fe700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78fe80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 -- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 msgr2=0x555c9a791550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:07.018 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 -- 172.21.15.40:0/2255496529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 msgr2=0x555c9a78fe80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:07.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78fe80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 -- 172.21.15.40:0/2255496529 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff89c1a2a20 con 0x555c9a77fbe0 2023-07-14T11:26:07.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff89affd700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:26:07.019 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff893fff700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78f940 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7ff884000c00 tx=0x7ff88400b9f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c746dce1dd869a94 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:07.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff87affd700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:26:07.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.006+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff8840056b0 con 0x555c9a77fbe0 2023-07-14T11:26:07.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ff88c099b20 con 0x555c9a77fbe0 2023-07-14T11:26:07.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(107..107 src has 1..107) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ff884029b40 con 0x555c9a77fbe0 2023-07-14T11:26:07.020 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff884029d20 con 0x555c9a77fbe0 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 10 client.26596 fetch_fsmap learned FSMap version 252 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x7ff89c10d210 con 0x555c9a77fbe0 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff88c099b20 con 0x555c9a77fbe0 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ff88403d070 con 0x555c9a77fbe0 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 10 client.26596 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:26:07.021 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 10 client.26596 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:26:07.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 20 client.26596 populate_metadata read hostname 'smithi040' 2023-07-14T11:26:07.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a7ef050 0x555c9a7f1460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:07.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 4 client.26596 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:26:07.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.007+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x555c9a68d050 con 0x555c9a7ef050 2023-07-14T11:26:07.022 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff89affd700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a7ef050 0x555c9a7f1460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:07.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff89affd700 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a7ef050 0x555c9a7f1460 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=65a47d538dafa524 server_cookie=8fdf1930406336fe in_seq=0 out_seq=0 2023-07-14T11:26:07.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff87affd700 10 client.26596 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:07.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff87affd700 1 -- 172.21.15.40:0/2255496529 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x555c9a68d050 con 0x555c9a7ef050 2023-07-14T11:26:07.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff87affd700 10 client.26596 handle_command_reply: tid=0 2023-07-14T11:26:07.023 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.008+0000 7ff8b52f7b80 1 client.26596 shutdown 2023-07-14T11:26:07.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 msgr2=0x555c9a78f940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:07.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78f940 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7ff884000c00 tx=0x7ff88400b9f0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 shutdown_connections 2023-07-14T11:26:07.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x555c9a7ef050 0x555c9a7f1460 crc :-1 s=READY pgs=126 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.024 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x555c9a78aad0 0x555c9a78fe80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x555c9a77f2d0 0x555c9a791550 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff87affd700 0 client.26596 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:07.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 --2- 172.21.15.40:0/2255496529 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x555c9a77fbe0 0x555c9a78f940 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 >> 172.21.15.40:0/2255496529 conn(0x555c9a775da0 msgr2=0x555c9a786fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:07.025 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 shutdown_connections 2023-07-14T11:26:07.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 1 -- 172.21.15.40:0/2255496529 wait complete. 2023-07-14T11:26:07.026 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8b52f7b80 20 client.26596 trim_cache size 0 max 16384 2023-07-14T11:26:07.026 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:26:07.027 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:26:07.028 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:26:07.028 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:26:07.028 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:26:07.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 msgr2=0x7ff8880a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:07.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 0x7ff8880a3770 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7ff89c19c730 tx=0x7ff89400b040 comp rx=0 tx=0).stop 2023-07-14T11:26:07.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 msgr2=0x7ff89c19ba30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:07.029 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.009+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7ff88c00e9a0 tx=0x7ff88c00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 shutdown_connections 2023-07-14T11:26:07.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8880a12c0 0x7ff8880a3770 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff89c105ae0 0x7ff89c195aa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff89c0ffad0 0x7ff89c19ba30 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.030 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 --2- 172.21.15.40:0/1270115464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff89c100480 0x7ff89c19bf70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:07.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 >> 172.21.15.40:0/1270115464 conn(0x7ff89c0fb5f0 msgr2=0x7ff89c0fbb50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:07.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 shutdown_connections 2023-07-14T11:26:07.031 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:07.010+0000 7ff8a19ff700 1 -- 172.21.15.40:0/1270115464 wait complete. 2023-07-14T11:26:07.032 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:26:08.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:07 smithi045 ceph-mon[128791]: pgmap v436: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:08.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:07 smithi130 ceph-mon[129405]: pgmap v436: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:07 smithi040 ceph-mon[97954]: pgmap v436: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:09 smithi130 ceph-mon[129405]: pgmap v437: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:09 smithi130 ceph-mon[129405]: osdmap e108: 12 total, 12 up, 12 in 2023-07-14T11:26:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:09 smithi040 ceph-mon[97954]: pgmap v437: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:09 smithi040 ceph-mon[97954]: osdmap e108: 12 total, 12 up, 12 in 2023-07-14T11:26:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:09 smithi045 ceph-mon[128791]: pgmap v437: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:26:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:09 smithi045 ceph-mon[128791]: osdmap e108: 12 total, 12 up, 12 in 2023-07-14T11:26:12.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:11 smithi130 ceph-mon[129405]: pgmap v439: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:12.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:11 smithi040 ceph-mon[97954]: pgmap v439: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:12.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:11 smithi045 ceph-mon[128791]: pgmap v439: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:14.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:13 smithi130 ceph-mon[129405]: pgmap v440: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:14.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:13 smithi040 ceph-mon[97954]: pgmap v440: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:14.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:13 smithi045 ceph-mon[128791]: pgmap v440: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:26:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:15 smithi130 ceph-mon[129405]: osdmap e109: 12 total, 12 up, 12 in 2023-07-14T11:26:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:15 smithi130 ceph-mon[129405]: pgmap v442: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:15 smithi040 ceph-mon[97954]: osdmap e109: 12 total, 12 up, 12 in 2023-07-14T11:26:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:15 smithi040 ceph-mon[97954]: pgmap v442: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:15 smithi045 ceph-mon[128791]: osdmap e109: 12 total, 12 up, 12 in 2023-07-14T11:26:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:15 smithi045 ceph-mon[128791]: pgmap v442: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:18.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:17 smithi130 ceph-mon[129405]: pgmap v443: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:18.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:17 smithi040 ceph-mon[97954]: pgmap v443: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:18.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:17 smithi045 ceph-mon[128791]: pgmap v443: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:26:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:19 smithi130 ceph-mon[129405]: pgmap v444: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:26:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:19 smithi040 ceph-mon[97954]: pgmap v444: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:26:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:19 smithi045 ceph-mon[128791]: pgmap v444: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:26:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:21 smithi130 ceph-mon[129405]: pgmap v445: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:21 smithi040 ceph-mon[97954]: pgmap v445: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:21 smithi045 ceph-mon[128791]: pgmap v445: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:23 smithi130 ceph-mon[129405]: pgmap v446: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:23 smithi040 ceph-mon[97954]: pgmap v446: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:23 smithi045 ceph-mon[128791]: pgmap v446: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:26:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:25 smithi130 ceph-mon[129405]: pgmap v447: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:26:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:25 smithi040 ceph-mon[97954]: pgmap v447: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:26:26.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:25 smithi045 ceph-mon[128791]: pgmap v447: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:26:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:27 smithi130 ceph-mon[129405]: pgmap v448: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:27 smithi040 ceph-mon[97954]: pgmap v448: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:27 smithi045 ceph-mon[128791]: pgmap v448: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:29 smithi130 ceph-mon[129405]: pgmap v449: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:29 smithi040 ceph-mon[97954]: pgmap v449: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:29 smithi045 ceph-mon[128791]: pgmap v449: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:32.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:31 smithi130 ceph-mon[129405]: pgmap v450: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:31 smithi040 ceph-mon[97954]: pgmap v450: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:32.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:31 smithi045 ceph-mon[128791]: pgmap v450: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:33 smithi130 ceph-mon[129405]: pgmap v451: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:33 smithi040 ceph-mon[97954]: pgmap v451: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:34.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:33 smithi045 ceph-mon[128791]: pgmap v451: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:35 smithi130 ceph-mon[129405]: pgmap v452: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:35 smithi040 ceph-mon[97954]: pgmap v452: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:36.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:35 smithi045 ceph-mon[128791]: pgmap v452: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:37.033 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:26:37.251 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 -- 172.21.15.40:0/382656254 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190105ae0 msgr2=0x7fa190105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 --2- 172.21.15.40:0/382656254 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190105ae0 0x7fa190105eb0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7fa18c009990 tx=0x7fa18c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 -- 172.21.15.40:0/382656254 shutdown_connections 2023-07-14T11:26:37.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 --2- 172.21.15.40:0/382656254 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa190100480 0x7fa19010c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 --2- 172.21.15.40:0/382656254 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190105ae0 0x7fa190105eb0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 --2- 172.21.15.40:0/382656254 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa1900ffad0 0x7fa1900fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 -- 172.21.15.40:0/382656254 >> 172.21.15.40:0/382656254 conn(0x7fa1900fb5f0 msgr2=0x7fa1900fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 -- 172.21.15.40:0/382656254 shutdown_connections 2023-07-14T11:26:37.253 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.249+0000 7fa197fff700 1 -- 172.21.15.40:0/382656254 wait complete. 2023-07-14T11:26:37.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 Processor -- start 2023-07-14T11:26:37.255 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 -- start start 2023-07-14T11:26:37.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 0x7fa190197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 0x7fa190198300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa19019cda0 con 0x7fa1900ffad0 2023-07-14T11:26:37.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa19019cf10 con 0x7fa190105ae0 2023-07-14T11:26:37.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.250+0000 7fa197fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa19019d0d0 con 0x7fa190100480 2023-07-14T11:26:37.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 0x7fa190197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 0x7fa190198300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:37948/0 (socket says 172.21.15.40:37948) 2023-07-14T11:26:37.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19659c700 1 -- 172.21.15.40:0/1154404000 learned_addr learned my addr 172.21.15.40:0/1154404000 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:37.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 -- 172.21.15.40:0/1154404000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 msgr2=0x7fa19019c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 -- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 msgr2=0x7fa190197dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.258 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 0x7fa190197dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 -- 172.21.15.40:0/1154404000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa18c0095f0 con 0x7fa190100480 2023-07-14T11:26:37.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19659c700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:26:37.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa195d9b700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 0x7fa190197dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:26:37.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.251+0000 7fa19559a700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 0x7fa190198300 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fa18000e9f0 tx=0x7fa18000ed00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.252+0000 7fa186ffd700 1 -- 172.21.15.40:0/1154404000 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18000c9c0 con 0x7fa190100480 2023-07-14T11:26:37.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.252+0000 7fa186ffd700 1 -- 172.21.15.40:0/1154404000 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa180010430 con 0x7fa190100480 2023-07-14T11:26:37.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.252+0000 7fa186ffd700 1 -- 172.21.15.40:0/1154404000 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa180018680 con 0x7fa190100480 2023-07-14T11:26:37.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.252+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa19019d3b0 con 0x7fa190100480 2023-07-14T11:26:37.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.252+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa1901a4bf0 con 0x7fa190100480 2023-07-14T11:26:37.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.255+0000 7fa186ffd700 1 -- 172.21.15.40:0/1154404000 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fa180010860 con 0x7fa190100480 2023-07-14T11:26:37.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.256+0000 7fa1ab910b80 1 Processor -- start 2023-07-14T11:26:37.263 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 -- start start 2023-07-14T11:26:37.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb106a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb1b590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb1deb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.264 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdea16cb0 con 0x55ccdeb1bad0 2023-07-14T11:26:37.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdea69a40 con 0x55ccdeb102d0 2023-07-14T11:26:37.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdea62ed0 con 0x55ccdeb10be0 2023-07-14T11:26:37.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa186ffd700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 0x7fa17c0a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.257+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb106a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.265 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb1b590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb106a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:37962/0 (socket says 172.21.15.40:37962) 2023-07-14T11:26:37.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa19659c700 1 -- 172.21.15.40:0/2548623222 learned_addr learned my addr 172.21.15.40:0/2548623222 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:37.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa19559a700 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb1deb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa186ffd700 1 -- 172.21.15.40:0/1154404000 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fa180014070 con 0x7fa190100480 2023-07-14T11:26:37.266 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 -- 172.21.15.40:0/2548623222 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 msgr2=0x55ccdeb106a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb106a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 -- 172.21.15.40:0/2548623222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 msgr2=0x55ccdeb1deb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb1deb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa19659c700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 0x7fa17c0a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.267 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 -- 172.21.15.40:0/2548623222 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ccde890de0 con 0x55ccdeb10be0 2023-07-14T11:26:37.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa19559a700 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb1deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:26:37.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa195d9b700 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb1b590 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fa18c000c00 tx=0x7fa18c005ea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=242a00a5727988bc server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa17bfff700 1 -- 172.21.15.40:0/2548623222 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c01d050 con 0x55ccdeb10be0 2023-07-14T11:26:37.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa17bfff700 1 -- 172.21.15.40:0/2548623222 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa18c01e040 con 0x55ccdeb10be0 2023-07-14T11:26:37.268 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.258+0000 7fa17bfff700 1 -- 172.21.15.40:0/2548623222 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18c005420 con 0x55ccdeb10be0 2023-07-14T11:26:37.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2548623222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 msgr2=0x55ccdeb1b590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa19659c700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 0x7fa17c0a37f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fa188007b60 tx=0x7fa188004b40 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb1b590 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7fa18c000c00 tx=0x7fa18c005ea0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2548623222 shutdown_connections 2023-07-14T11:26:37.269 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb1deb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb106a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2548623222 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb1b590 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2548623222 >> 172.21.15.40:0/2548623222 conn(0x55ccdeb06da0 msgr2=0x55ccdeb0c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.270 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2548623222 shutdown_connections 2023-07-14T11:26:37.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.259+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2548623222 wait complete. 2023-07-14T11:26:37.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 -- start start 2023-07-14T11:26:37.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 Processor -- start 2023-07-14T11:26:37.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b520 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.271 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18e70 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdea02ac0 con 0x55ccdeb102d0 2023-07-14T11:26:37.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdea027c0 con 0x55ccdeb10be0 2023-07-14T11:26:37.272 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.260+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccde8e6920 con 0x55ccdeb1bad0 2023-07-14T11:26:37.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa1ab910b80 10 client.0 _ll_register_callbacks cb 0x55ccdea00760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:26:37.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.273 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:37964/0 (socket says 172.21.15.40:37964) 2023-07-14T11:26:37.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 -- 172.21.15.40:0/503754977 learned_addr learned my addr 172.21.15.40:0/503754977 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:37.274 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19659c700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 -- 172.21.15.40:0/503754977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb1bad0 msgr2=0x55ccdeb18e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 -- 172.21.15.40:0/503754977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb102d0 msgr2=0x55ccdeb1b520 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b520 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 -- 172.21.15.40:0/503754977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa19010d210 con 0x55ccdeb10be0 2023-07-14T11:26:37.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa19559a700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fa1800ca590 tx=0x7fa1800caa90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=e39b596e49aedf66 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa1797fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:26:37.280 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.261+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18009dee0 con 0x55ccdeb10be0 2023-07-14T11:26:37.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55ccde9bc4f0 con 0x55ccdeb10be0 2023-07-14T11:26:37.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fa1800a2300 con 0x55ccdeb10be0 2023-07-14T11:26:37.281 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa18009a110 con 0x55ccdeb10be0 2023-07-14T11:26:37.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 10 client.26645 fetch_fsmap learned FSMap version 252 2023-07-14T11:26:37.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x55ccde879c60 con 0x55ccdeb10be0 2023-07-14T11:26:37.282 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa1800b6080 con 0x55ccdeb10be0 2023-07-14T11:26:37.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fa1800adbb0 con 0x55ccdeb10be0 2023-07-14T11:26:37.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 10 client.26645 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:26:37.283 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 10 client.26645 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:26:37.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 20 client.26645 populate_metadata read hostname 'smithi040' 2023-07-14T11:26:37.284 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb2b300 0x55ccdeb2d710 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 4 client.26645 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:26:37.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.262+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55ccdea5beb0 con 0x55ccdeb2b300 2023-07-14T11:26:37.285 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.263+0000 7fa195d9b700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb2b300 0x55ccdeb2d710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.263+0000 7fa195d9b700 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb2b300 0x55ccdeb2d710 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=47306afe5ac0370a server_cookie=59913cc9b61852dd in_seq=0 out_seq=0 2023-07-14T11:26:37.286 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.263+0000 7fa1797fa700 10 client.26645 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:37.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.264+0000 7fa1797fa700 1 -- 172.21.15.40:0/503754977 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55ccdea5beb0 con 0x55ccdeb2b300 2023-07-14T11:26:37.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.264+0000 7fa1797fa700 10 client.26645 handle_command_reply: tid=0 2023-07-14T11:26:37.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.264+0000 7fa1ab910b80 1 client.26645 shutdown 2023-07-14T11:26:37.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 msgr2=0x55ccdeb18730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.288 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7fa1800ca590 tx=0x7fa1800caa90 comp rx=0 tx=0).stop 2023-07-14T11:26:37.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 shutdown_connections 2023-07-14T11:26:37.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb2b300 0x55ccdeb2d710 crc :-1 s=READY pgs=127 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/503754977 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb10be0 0x55ccdeb18730 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 >> 172.21.15.40:0/503754977 conn(0x55ccdeb06da0 msgr2=0x55ccdeb0c290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1797fa700 0 client.26645 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:37.291 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 shutdown_connections 2023-07-14T11:26:37.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 1 -- 172.21.15.40:0/503754977 wait complete. 2023-07-14T11:26:37.292 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.265+0000 7fa1ab910b80 20 client.26645 trim_cache size 0 max 16384 2023-07-14T11:26:37.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.282+0000 7fa1ab910b80 1 Processor -- start 2023-07-14T11:26:37.299 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.282+0000 7fa1ab910b80 1 -- start start 2023-07-14T11:26:37.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b050 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 0x55ccdeb17ff0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.300 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18530 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb11330 con 0x55ccdeb10be0 2023-07-14T11:26:37.301 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb121a0 con 0x55ccdeb1bad0 2023-07-14T11:26:37.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb11dd0 con 0x55ccdeb102d0 2023-07-14T11:26:37.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 0x55ccdeb17ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.302 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.303 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:37974/0 (socket says 172.21.15.40:37974) 2023-07-14T11:26:37.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44664/0 (socket says 172.21.15.40:44664) 2023-07-14T11:26:37.304 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19659c700 1 -- 172.21.15.40:0/4038487664 learned_addr learned my addr 172.21.15.40:0/4038487664 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:37.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 -- 172.21.15.40:0/4038487664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 msgr2=0x55ccdeb1b050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 -- 172.21.15.40:0/4038487664 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 msgr2=0x55ccdeb18530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.305 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.283+0000 7fa19559a700 1 -- 172.21.15.40:0/4038487664 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55ccde8b7250 con 0x55ccdeb10be0 2023-07-14T11:26:37.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa19559a700 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 0x55ccdeb17ff0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7fa180005c30 tx=0x7fa18000e940 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8156917cafac504 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa184ff9700 1 -- 172.21.15.40:0/4038487664 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1800a2300 con 0x55ccdeb10be0 2023-07-14T11:26:37.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa184ff9700 1 -- 172.21.15.40:0/4038487664 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55ccde9bc4f0 con 0x55ccdeb10be0 2023-07-14T11:26:37.306 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa184ff9700 1 -- 172.21.15.40:0/4038487664 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18009a110 con 0x55ccdeb10be0 2023-07-14T11:26:37.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 -- 172.21.15.40:0/4038487664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 msgr2=0x55ccdeb17ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 0x55ccdeb17ff0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 crypto rx=0x7fa180005c30 tx=0x7fa18000e940 comp rx=0 tx=0).stop 2023-07-14T11:26:37.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 -- 172.21.15.40:0/4038487664 shutdown_connections 2023-07-14T11:26:37.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb18530 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.307 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb102d0 0x55ccdeb1b050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/4038487664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb10be0 0x55ccdeb17ff0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 -- 172.21.15.40:0/4038487664 >> 172.21.15.40:0/4038487664 conn(0x55ccdeb06da0 msgr2=0x55ccdeb13a90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.284+0000 7fa1ab910b80 1 -- 172.21.15.40:0/4038487664 shutdown_connections 2023-07-14T11:26:37.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.285+0000 7fa1ab910b80 1 -- 172.21.15.40:0/4038487664 wait complete. 2023-07-14T11:26:37.308 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.285+0000 7fa1ab910b80 1 -- start start 2023-07-14T11:26:37.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.285+0000 7fa1ab910b80 1 Processor -- start 2023-07-14T11:26:37.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.285+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb224e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb20e00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.309 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb214d0 con 0x55ccdeb1bad0 2023-07-14T11:26:37.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb21640 con 0x55ccdeb102d0 2023-07-14T11:26:37.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55ccdeb7f1d0 con 0x55ccdeb10be0 2023-07-14T11:26:37.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1ab910b80 10 client.0 _ll_register_callbacks cb 0x7fa180099ca0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:26:37.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.310 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb224e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:44668/0 (socket says 172.21.15.40:44668) 2023-07-14T11:26:37.311 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa195d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb224e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:37984/0 (socket says 172.21.15.40:37984) 2023-07-14T11:26:37.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 -- 172.21.15.40:0/2203697147 learned_addr learned my addr 172.21.15.40:0/2203697147 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:26:37.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19659c700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb20e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.312 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 -- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 msgr2=0x55ccdeb224e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb224e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.313 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 -- 172.21.15.40:0/2203697147 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 msgr2=0x55ccdeb20e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:26:37.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb20e00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.314 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 -- 172.21.15.40:0/2203697147 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55ccde879c60 con 0x55ccdeb10be0 2023-07-14T11:26:37.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa19559a700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fa180098200 tx=0x7fa1800adde0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=d4db596d462e4cfb server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:26:37.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.286+0000 7fa1797fa700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:26:37.315 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa18009a310 con 0x55ccdeb10be0 2023-07-14T11:26:37.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fa1800a2300 con 0x55ccdeb10be0 2023-07-14T11:26:37.316 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7fa18c01e040 con 0x55ccdeb10be0 2023-07-14T11:26:37.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7fa1800961f0 con 0x55ccdeb10be0 2023-07-14T11:26:37.317 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 10 client.26614 fetch_fsmap learned FSMap version 252 2023-07-14T11:26:37.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x55ccde88f2b0 con 0x55ccdeb10be0 2023-07-14T11:26:37.318 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fa18c01e040 con 0x55ccdeb10be0 2023-07-14T11:26:37.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7fa180095ed0 con 0x55ccdeb10be0 2023-07-14T11:26:37.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 10 client.26614 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:26:37.319 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 10 client.26614 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:26:37.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 20 client.26614 populate_metadata read hostname 'smithi040' 2023-07-14T11:26:37.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb80360 0x55ccdeb82740 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:26:37.320 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 4 client.26614 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:26:37.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55ccdea1e050 con 0x55ccdeb80360 2023-07-14T11:26:37.321 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.287+0000 7fa195d9b700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb80360 0x55ccdeb82740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:26:37.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.288+0000 7fa195d9b700 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb80360 0x55ccdeb82740 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c0451f9d732bd017 server_cookie=7a770d19d21fdd61 in_seq=0 out_seq=0 2023-07-14T11:26:37.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.288+0000 7fa1797fa700 10 client.26614 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:37.322 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.288+0000 7fa1797fa700 1 -- 172.21.15.40:0/2203697147 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55ccdea1e050 con 0x55ccdeb80360 2023-07-14T11:26:37.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.288+0000 7fa1797fa700 10 client.26614 handle_command_reply: tid=0 2023-07-14T11:26:37.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.288+0000 7fa1ab910b80 1 client.26614 shutdown 2023-07-14T11:26:37.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 msgr2=0x55ccdeb208c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.323 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fa180098200 tx=0x7fa1800adde0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 shutdown_connections 2023-07-14T11:26:37.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55ccdeb80360 0x55ccdeb82740 crc :-1 s=READY pgs=128 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55ccdeb1bad0 0x55ccdeb20e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1797fa700 0 client.26614 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:26:37.324 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55ccdeb102d0 0x55ccdeb224e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 --2- 172.21.15.40:0/2203697147 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55ccdeb10be0 0x55ccdeb208c0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 >> 172.21.15.40:0/2203697147 conn(0x55ccdeb06da0 msgr2=0x55ccdeb17ff0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 shutdown_connections 2023-07-14T11:26:37.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 1 -- 172.21.15.40:0/2203697147 wait complete. 2023-07-14T11:26:37.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa1ab910b80 20 client.26614 trim_cache size 0 max 16384 2023-07-14T11:26:37.326 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:26:37.326 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:26:37.326 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:26:37.327 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:26:37.327 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:26:37.327 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:26:37.327 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:26:37.327 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:26:37.328 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:26:37.328 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:26:37.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 msgr2=0x7fa17c0a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.329 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 0x7fa17c0a37f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fa188007b60 tx=0x7fa188004b40 comp rx=0 tx=0).stop 2023-07-14T11:26:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 msgr2=0x7fa190198300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:26:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.289+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 0x7fa190198300 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7fa18000e9f0 tx=0x7fa18000ed00 comp rx=0 tx=0).stop 2023-07-14T11:26:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 shutdown_connections 2023-07-14T11:26:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fa17c0a1340 0x7fa17c0a37f0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.330 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fa190105ae0 0x7fa19019c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa1900ffad0 0x7fa190197dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 --2- 172.21.15.40:0/1154404000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fa190100480 0x7fa190198300 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:26:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 >> 172.21.15.40:0/1154404000 conn(0x7fa1900fb5f0 msgr2=0x7fa1900fbbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:26:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 shutdown_connections 2023-07-14T11:26:37.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:26:37.290+0000 7fa197fff700 1 -- 172.21.15.40:0/1154404000 wait complete. 2023-07-14T11:26:37.332 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:26:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:37 smithi130 ceph-mon[129405]: pgmap v453: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:38.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:37 smithi040 ceph-mon[97954]: pgmap v453: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:38.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:37 smithi045 ceph-mon[128791]: pgmap v453: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:39 smithi130 ceph-mon[129405]: pgmap v454: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:39 smithi040 ceph-mon[97954]: pgmap v454: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:40.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:39 smithi045 ceph-mon[128791]: pgmap v454: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:42.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:41 smithi130 ceph-mon[129405]: pgmap v455: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:42.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:41 smithi040 ceph-mon[97954]: pgmap v455: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:42.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:41 smithi045 ceph-mon[128791]: pgmap v455: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:44.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:43 smithi130 ceph-mon[129405]: pgmap v456: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:43 smithi040 ceph-mon[97954]: pgmap v456: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:44.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:43 smithi045 ceph-mon[128791]: pgmap v456: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:45 smithi130 ceph-mon[129405]: pgmap v457: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:45 smithi040 ceph-mon[97954]: pgmap v457: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:45 smithi045 ceph-mon[128791]: pgmap v457: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:48.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:47 smithi130 ceph-mon[129405]: pgmap v458: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:48.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:47 smithi040 ceph-mon[97954]: pgmap v458: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:48.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:47 smithi045 ceph-mon[128791]: pgmap v458: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:50.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:49 smithi130 ceph-mon[129405]: pgmap v459: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:49 smithi040 ceph-mon[97954]: pgmap v459: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:50.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:49 smithi045 ceph-mon[128791]: pgmap v459: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:51 smithi130 ceph-mon[129405]: pgmap v460: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:51 smithi040 ceph-mon[97954]: pgmap v460: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:51 smithi045 ceph-mon[128791]: pgmap v460: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:54.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:53 smithi130 ceph-mon[129405]: pgmap v461: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:53 smithi040 ceph-mon[97954]: pgmap v461: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:54.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:53 smithi045 ceph-mon[128791]: pgmap v461: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:55 smithi130 ceph-mon[129405]: pgmap v462: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:55 smithi040 ceph-mon[97954]: pgmap v462: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:55 smithi045 ceph-mon[128791]: pgmap v462: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:26:58.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:57 smithi130 ceph-mon[129405]: pgmap v463: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:58.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:57 smithi040 ceph-mon[97954]: pgmap v463: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:26:58.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:57 smithi045 ceph-mon[128791]: pgmap v463: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:26:59 smithi130 ceph-mon[129405]: pgmap v464: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:26:59 smithi040 ceph-mon[97954]: pgmap v464: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:00.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:26:59 smithi045 ceph-mon[128791]: pgmap v464: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:01.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:00 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:27:01.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:00 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:27:01.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:00 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:27:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:01 smithi130 ceph-mon[129405]: pgmap v465: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:01 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:27:02.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:01 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:27:02.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:01 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:27:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:01 smithi040 ceph-mon[97954]: pgmap v465: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:02.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:01 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:27:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:01 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:27:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:01 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:27:02.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:01 smithi045 ceph-mon[128791]: pgmap v465: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:01 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:27:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:01 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:27:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:01 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:27:04.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:03 smithi130 ceph-mon[129405]: pgmap v466: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:03 smithi040 ceph-mon[97954]: pgmap v466: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:04.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:03 smithi045 ceph-mon[128791]: pgmap v466: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:05 smithi130 ceph-mon[129405]: pgmap v467: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:27:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:05 smithi040 ceph-mon[97954]: pgmap v467: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:27:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:05 smithi045 ceph-mon[128791]: pgmap v467: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:27:07.332 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:27:07.569 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/3243874997 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580728b0 msgr2=0x7ffb58072c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/3243874997 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580728b0 0x7ffb58072c80 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7ffb48009990 tx=0x7ffb48009ca0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/3243874997 shutdown_connections 2023-07-14T11:27:07.570 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/3243874997 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5810e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/3243874997 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580728b0 0x7ffb58072c80 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/3243874997 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580731c0 0x7ffb5810bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/3243874997 >> 172.21.15.40:0/3243874997 conn(0x7ffb580fb5f0 msgr2=0x7ffb580fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/3243874997 shutdown_connections 2023-07-14T11:27:07.571 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.566+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/3243874997 wait complete. 2023-07-14T11:27:07.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.567+0000 7ffb5d1ff700 1 Processor -- start 2023-07-14T11:27:07.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.567+0000 7ffb5d1ff700 1 -- start start 2023-07-14T11:27:07.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.567+0000 7ffb5d1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580728b0 0x7ffb58197de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.573 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.567+0000 7ffb5d1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.567+0000 7ffb5d1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5819c780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5d1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb5819cdc0 con 0x7ffb580728b0 2023-07-14T11:27:07.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5d1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb5819cf30 con 0x7ffb580731c0 2023-07-14T11:27:07.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5d1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffb5819d0f0 con 0x7ffb5810c310 2023-07-14T11:27:07.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580728b0 0x7ffb58197de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38104/0 (socket says 172.21.15.40:38104) 2023-07-14T11:27:07.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 -- 172.21.15.40:0/384211227 learned_addr learned my addr 172.21.15.40:0/384211227 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:07.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5759e700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5819c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.575 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 -- 172.21.15.40:0/384211227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 msgr2=0x7ffb5819c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5819c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 -- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580728b0 msgr2=0x7ffb58197de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580728b0 0x7ffb58197de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 -- 172.21.15.40:0/384211227 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffb50009c40 con 0x7ffb580731c0 2023-07-14T11:27:07.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5759e700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5819c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.568+0000 7ffb5659c700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c00e9a0 tx=0x7ffb4c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.569+0000 7ffb3ffff700 1 -- 172.21.15.40:0/384211227 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c009790 con 0x7ffb580731c0 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.569+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffb480095f0 con 0x7ffb580731c0 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.569+0000 7ffb3ffff700 1 -- 172.21.15.40:0/384211227 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb4c00ce80 con 0x7ffb580731c0 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.569+0000 7ffb3ffff700 1 -- 172.21.15.40:0/384211227 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c018680 con 0x7ffb580731c0 2023-07-14T11:27:07.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.569+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb581a4a30 con 0x7ffb580731c0 2023-07-14T11:27:07.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.572+0000 7ffb3ffff700 1 -- 172.21.15.40:0/384211227 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7ffb4c010430 con 0x7ffb580731c0 2023-07-14T11:27:07.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.573+0000 7ffb70b64b80 1 Processor -- start 2023-07-14T11:27:07.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 -- start start 2023-07-14T11:27:07.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc63eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efb5ccb0 con 0x55c5efc61ad0 2023-07-14T11:27:07.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efbafa40 con 0x55c5efc562d0 2023-07-14T11:27:07.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.574+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efba8ed0 con 0x55c5efc56be0 2023-07-14T11:27:07.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5759e700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5759e700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38120/0 (socket says 172.21.15.40:38120) 2023-07-14T11:27:07.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5759e700 1 -- 172.21.15.40:0/274997538 learned_addr learned my addr 172.21.15.40:0/274997538 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:07.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5659c700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc63eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb3ffff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 0x7ffb400a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 -- 172.21.15.40:0/274997538 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 msgr2=0x55c5efc566a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 -- 172.21.15.40:0/274997538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 msgr2=0x55c5efc63eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc63eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 -- 172.21.15.40:0/274997538 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c5ef9d6de0 con 0x55c5efc56be0 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb3ffff700 1 -- 172.21.15.40:0/384211227 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ffb4c014070 con 0x7ffb580731c0 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5759e700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb5759e700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 0x7ffb400a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.575+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61590 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7ffb48000c00 tx=0x7ffb48005ea0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=46de252ea93eda3b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb3d7fa700 1 -- 172.21.15.40:0/274997538 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4801d070 con 0x55c5efc56be0 2023-07-14T11:27:07.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb3d7fa700 1 -- 172.21.15.40:0/274997538 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb4801e040 con 0x55c5efc56be0 2023-07-14T11:27:07.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb3d7fa700 1 -- 172.21.15.40:0/274997538 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb48004860 con 0x55c5efc56be0 2023-07-14T11:27:07.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 -- 172.21.15.40:0/274997538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 msgr2=0x55c5efc61590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb5759e700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 0x7ffb400a3770 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7ffb50007a00 tx=0x7ffb50004ad0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61590 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7ffb48000c00 tx=0x7ffb48005ea0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 -- 172.21.15.40:0/274997538 shutdown_connections 2023-07-14T11:27:07.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc63eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc566a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/274997538 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61590 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.576+0000 7ffb70b64b80 1 -- 172.21.15.40:0/274997538 >> 172.21.15.40:0/274997538 conn(0x55c5efc4cda0 msgr2=0x55c5efc526a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.577+0000 7ffb70b64b80 1 -- 172.21.15.40:0/274997538 shutdown_connections 2023-07-14T11:27:07.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.577+0000 7ffb70b64b80 1 -- 172.21.15.40:0/274997538 wait complete. 2023-07-14T11:27:07.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.577+0000 7ffb70b64b80 1 -- start start 2023-07-14T11:27:07.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.577+0000 7ffb70b64b80 1 Processor -- start 2023-07-14T11:27:07.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc5e770 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5eda0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efb48ac0 con 0x55c5efc61ad0 2023-07-14T11:27:07.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efb487c0 con 0x55c5efc562d0 2023-07-14T11:27:07.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efa2c920 con 0x55c5efc56be0 2023-07-14T11:27:07.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb70b64b80 10 client.0 _ll_register_callbacks cb 0x55c5efb46760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:27:07.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38136/0 (socket says 172.21.15.40:38136) 2023-07-14T11:27:07.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb56d9d700 1 -- 172.21.15.40:0/3704100883 learned_addr learned my addr 172.21.15.40:0/3704100883 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:07.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc5e770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5759e700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5eda0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 -- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 msgr2=0x55c5efc614f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 -- 172.21.15.40:0/3704100883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 msgr2=0x55c5efc5eda0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5eda0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb5659c700 1 -- 172.21.15.40:0/3704100883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffb500092b0 con 0x55c5efc56be0 2023-07-14T11:27:07.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.578+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb5659c700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc5e770 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c0ca630 tx=0x7ffb4c0cab90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=dd482ac9e4c52fdc server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:27:07.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c09a410 con 0x55c5efc56be0 2023-07-14T11:27:07.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c5efb024f0 con 0x55c5efc56be0 2023-07-14T11:27:07.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ffb4c09dfe0 con 0x55c5efc56be0 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7ffb4c0a2400 con 0x55c5efc56be0 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 10 client.26626 fetch_fsmap learned FSMap version 252 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x7ffb5810f390 con 0x55c5efc56be0 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c5efb024f0 con 0x55c5efc56be0 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ffb4c095da0 con 0x55c5efc56be0 2023-07-14T11:27:07.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 10 client.26626 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:27:07.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 10 client.26626 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:27:07.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 20 client.26626 populate_metadata read hostname 'smithi040' 2023-07-14T11:27:07.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.579+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efc71320 0x55c5efc73730 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.580+0000 7ffb70b64b80 4 client.26626 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:27:07.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.580+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c5efba1eb0 con 0x55c5efc71320 2023-07-14T11:27:07.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.580+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efc71320 0x55c5efc73730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.580+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efc71320 0x55c5efc73730 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=9f362a8dbd7165e7 server_cookie=77a1c88f5e672af0 in_seq=0 out_seq=0 2023-07-14T11:27:07.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.580+0000 7ffb3aff5700 10 client.26626 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:07.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.581+0000 7ffb3aff5700 1 -- 172.21.15.40:0/3704100883 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c5efba1eb0 con 0x55c5efc71320 2023-07-14T11:27:07.597 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.581+0000 7ffb3aff5700 10 client.26626 handle_command_reply: tid=0 2023-07-14T11:27:07.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 client.26626 shutdown 2023-07-14T11:27:07.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 msgr2=0x55c5efc5e770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc5e770 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c0ca630 tx=0x7ffb4c0cab90 comp rx=0 tx=0).stop 2023-07-14T11:27:07.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 shutdown_connections 2023-07-14T11:27:07.598 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efc71320 0x55c5efc73730 crc :-1 s=READY pgs=129 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5eda0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb3aff5700 0 client.26626 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:07.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc614f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/3704100883 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc5e770 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.599 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 >> 172.21.15.40:0/3704100883 conn(0x55c5efc4cda0 msgr2=0x55c5efc52230 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 shutdown_connections 2023-07-14T11:27:07.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 1 -- 172.21.15.40:0/3704100883 wait complete. 2023-07-14T11:27:07.600 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.582+0000 7ffb70b64b80 20 client.26626 trim_cache size 0 max 16384 2023-07-14T11:27:07.603 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 Processor -- start 2023-07-14T11:27:07.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 -- start start 2023-07-14T11:27:07.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61020 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.604 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5b9e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc57d60 con 0x55c5efc61ad0 2023-07-14T11:27:07.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc68a20 con 0x55c5efc562d0 2023-07-14T11:27:07.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc593f0 con 0x55c5efc56be0 2023-07-14T11:27:07.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.605 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38146/0 (socket says 172.21.15.40:38146) 2023-07-14T11:27:07.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb5659c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb56d9d700 1 -- 172.21.15.40:0/2356389787 learned_addr learned my addr 172.21.15.40:0/2356389787 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:07.606 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.600+0000 7ffb5759e700 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5b9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.608 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 -- 172.21.15.40:0/2356389787 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 msgr2=0x55c5efc684e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 -- 172.21.15.40:0/2356389787 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 msgr2=0x55c5efc5b9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5b9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.609 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 -- 172.21.15.40:0/2356389787 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c5ef9bfc60 con 0x55c5efc56be0 2023-07-14T11:27:07.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb5659c700 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61020 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c0964b0 tx=0x7ffb4c010920 comp rx=0 tx=0).ready entity=mon.1 client_cookie=6a235aad45a0721f server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb3dffb700 1 -- 172.21.15.40:0/2356389787 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c0996a0 con 0x55c5efc56be0 2023-07-14T11:27:07.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb3dffb700 1 -- 172.21.15.40:0/2356389787 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb4c004e50 con 0x55c5efc56be0 2023-07-14T11:27:07.610 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb3dffb700 1 -- 172.21.15.40:0/2356389787 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb4c096000 con 0x55c5efc56be0 2023-07-14T11:27:07.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.601+0000 7ffb70b64b80 1 -- 172.21.15.40:0/2356389787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 msgr2=0x55c5efc61020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61020 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c0964b0 tx=0x7ffb4c010920 comp rx=0 tx=0).stop 2023-07-14T11:27:07.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 -- 172.21.15.40:0/2356389787 shutdown_connections 2023-07-14T11:27:07.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc61ad0 0x55c5efc5b9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.611 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc562d0 0x55c5efc684e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/2356389787 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc56be0 0x55c5efc61020 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 -- 172.21.15.40:0/2356389787 >> 172.21.15.40:0/2356389787 conn(0x55c5efc4cda0 msgr2=0x55c5efc59a30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 -- 172.21.15.40:0/2356389787 shutdown_connections 2023-07-14T11:27:07.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 -- 172.21.15.40:0/2356389787 wait complete. 2023-07-14T11:27:07.612 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 -- start start 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.602+0000 7ffb70b64b80 1 Processor -- start 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 0x55c5efc68480 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc66b80 con 0x55c5efc56be0 2023-07-14T11:27:07.613 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc66cf0 con 0x55c5efc61ad0 2023-07-14T11:27:07.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c5efc66eb0 con 0x55c5efc562d0 2023-07-14T11:27:07.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb70b64b80 10 client.0 _ll_register_callbacks cb 0x55c5efb46760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:27:07.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb5759e700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.614 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb5759e700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:38152/0 (socket says 172.21.15.40:38152) 2023-07-14T11:27:07.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb5759e700 1 -- 172.21.15.40:0/1432731298 learned_addr learned my addr 172.21.15.40:0/1432731298 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:07.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb56d9d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:33802/0 (socket says 172.21.15.40:33802) 2023-07-14T11:27:07.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.603+0000 7ffb5659c700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 0x55c5efc68480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 -- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 msgr2=0x55c5efc66470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.615 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 -- 172.21.15.40:0/1432731298 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 msgr2=0x55c5efc68480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:07.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 0x55c5efc68480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 -- 172.21.15.40:0/1432731298 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c5ef9d6de0 con 0x55c5efc562d0 2023-07-14T11:27:07.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb5759e700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.616 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb5659c700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 0x55c5efc68480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:07.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb56d9d700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7ffb48000c00 tx=0x7ffb480058e0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=12413acca7f1f8fd server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:07.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb3aff5700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:27:07.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ffb48039030 con 0x55c5efc562d0 2023-07-14T11:27:07.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(109..109 src has 1..109) v4 ==== 9432+0+0 (secure 0 0 0) 0x7ffb4803a070 con 0x55c5efc562d0 2023-07-14T11:27:07.617 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7ffb4801e040 con 0x55c5efc562d0 2023-07-14T11:27:07.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7ffb4801d040 con 0x55c5efc562d0 2023-07-14T11:27:07.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb70b64b80 10 client.26632 fetch_fsmap learned FSMap version 252 2023-07-14T11:27:07.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.604+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x7ffb5810f390 con 0x55c5efc562d0 2023-07-14T11:27:07.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ffb4801e040 con 0x55c5efc562d0 2023-07-14T11:27:07.618 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7ffb4800f770 con 0x55c5efc562d0 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 10 client.26632 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 10 client.26632 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 20 client.26632 populate_metadata read hostname 'smithi040' 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efcc60e0 0x55c5efcc84c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 4 client.26632 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:27:07.619 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c5efb64050 con 0x55c5efcc60e0 2023-07-14T11:27:07.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.605+0000 7ffb5659c700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efcc60e0 0x55c5efcc84c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:07.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb5659c700 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efcc60e0 0x55c5efcc84c0 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=b032b6ff62506648 server_cookie=95a81d04497415db in_seq=0 out_seq=0 2023-07-14T11:27:07.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb3aff5700 10 client.26632 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:07.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb3aff5700 1 -- 172.21.15.40:0/1432731298 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55c5efb64050 con 0x55c5efcc60e0 2023-07-14T11:27:07.621 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb3aff5700 10 client.26632 handle_command_reply: tid=0 2023-07-14T11:27:07.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 client.26632 shutdown 2023-07-14T11:27:07.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 msgr2=0x55c5efc67f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7ffb48000c00 tx=0x7ffb480058e0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 shutdown_connections 2023-07-14T11:27:07.622 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c5efcc60e0 0x55c5efcc84c0 crc :-1 s=READY pgs=130 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c5efc61ad0 0x55c5efc66470 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c5efc562d0 0x55c5efc67f40 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 --2- 172.21.15.40:0/1432731298 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c5efc56be0 0x55c5efc68480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 >> 172.21.15.40:0/1432731298 conn(0x55c5efc4cda0 msgr2=0x55c5efc583e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.623 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb3aff5700 0 client.26632 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:07.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.606+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 shutdown_connections 2023-07-14T11:27:07.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb70b64b80 1 -- 172.21.15.40:0/1432731298 wait complete. 2023-07-14T11:27:07.624 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb70b64b80 20 client.26632 trim_cache size 0 max 16384 2023-07-14T11:27:07.624 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:27:07.625 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:27:07.625 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:27:07.625 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:27:07.625 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:27:07.625 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:27:07.626 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:27:07.626 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:27:07.626 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:27:07.626 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:27:07.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 msgr2=0x7ffb400a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 0x7ffb400a3770 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7ffb50007a00 tx=0x7ffb50004ad0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.627 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 msgr2=0x7ffb58198320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:07.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7ffb4c00e9a0 tx=0x7ffb4c00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 shutdown_connections 2023-07-14T11:27:07.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ffb400a12c0 0x7ffb400a3770 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ffb5810c310 0x7ffb5819c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.628 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffb580728b0 0x7ffb58197de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 --2- 172.21.15.40:0/384211227 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ffb580731c0 0x7ffb58198320 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:07.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 >> 172.21.15.40:0/384211227 conn(0x7ffb580fb5f0 msgr2=0x7ffb5810e450 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:07.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 shutdown_connections 2023-07-14T11:27:07.629 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:07.607+0000 7ffb5d1ff700 1 -- 172.21.15.40:0/384211227 wait complete. 2023-07-14T11:27:07.630 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:27:08.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:07 smithi130 ceph-mon[129405]: pgmap v468: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:07 smithi040 ceph-mon[97954]: pgmap v468: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:07 smithi045 ceph-mon[128791]: pgmap v468: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:09 smithi130 ceph-mon[129405]: pgmap v469: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:10.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:09 smithi130 ceph-mon[129405]: osdmap e110: 12 total, 12 up, 12 in 2023-07-14T11:27:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:09 smithi040 ceph-mon[97954]: pgmap v469: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:09 smithi040 ceph-mon[97954]: osdmap e110: 12 total, 12 up, 12 in 2023-07-14T11:27:10.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:09 smithi045 ceph-mon[128791]: pgmap v469: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:27:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:09 smithi045 ceph-mon[128791]: osdmap e110: 12 total, 12 up, 12 in 2023-07-14T11:27:12.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:11 smithi130 ceph-mon[129405]: pgmap v471: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:12.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:11 smithi040 ceph-mon[97954]: pgmap v471: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:12.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:11 smithi045 ceph-mon[128791]: pgmap v471: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:14.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:13 smithi130 ceph-mon[129405]: pgmap v472: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:14.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:13 smithi040 ceph-mon[97954]: pgmap v472: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:14.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:13 smithi045 ceph-mon[128791]: pgmap v472: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:27:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:15 smithi130 ceph-mon[129405]: osdmap e111: 12 total, 12 up, 12 in 2023-07-14T11:27:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:15 smithi130 ceph-mon[129405]: pgmap v474: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:15 smithi040 ceph-mon[97954]: osdmap e111: 12 total, 12 up, 12 in 2023-07-14T11:27:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:15 smithi040 ceph-mon[97954]: pgmap v474: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:15 smithi045 ceph-mon[128791]: osdmap e111: 12 total, 12 up, 12 in 2023-07-14T11:27:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:15 smithi045 ceph-mon[128791]: pgmap v474: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:18.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:17 smithi130 ceph-mon[129405]: pgmap v475: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:18.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:17 smithi040 ceph-mon[97954]: pgmap v475: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:18.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:17 smithi045 ceph-mon[128791]: pgmap v475: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:27:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:19 smithi130 ceph-mon[129405]: pgmap v476: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:27:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:19 smithi040 ceph-mon[97954]: pgmap v476: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:27:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:19 smithi045 ceph-mon[128791]: pgmap v476: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:27:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:21 smithi130 ceph-mon[129405]: pgmap v477: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:21 smithi040 ceph-mon[97954]: pgmap v477: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:22.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:21 smithi045 ceph-mon[128791]: pgmap v477: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:23 smithi130 ceph-mon[129405]: pgmap v478: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:23 smithi040 ceph-mon[97954]: pgmap v478: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:23 smithi045 ceph-mon[128791]: pgmap v478: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:27:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:25 smithi130 ceph-mon[129405]: pgmap v479: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:27:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:25 smithi040 ceph-mon[97954]: pgmap v479: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:27:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:25 smithi045 ceph-mon[128791]: pgmap v479: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:27:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:27 smithi130 ceph-mon[129405]: pgmap v480: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:27 smithi040 ceph-mon[97954]: pgmap v480: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:27 smithi045 ceph-mon[128791]: pgmap v480: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:29 smithi130 ceph-mon[129405]: pgmap v481: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:29 smithi040 ceph-mon[97954]: pgmap v481: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:29 smithi045 ceph-mon[128791]: pgmap v481: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:32.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:31 smithi130 ceph-mon[129405]: pgmap v482: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:31 smithi040 ceph-mon[97954]: pgmap v482: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:32.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:31 smithi045 ceph-mon[128791]: pgmap v482: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:33 smithi130 ceph-mon[129405]: pgmap v483: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:33 smithi040 ceph-mon[97954]: pgmap v483: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:33 smithi045 ceph-mon[128791]: pgmap v483: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:35 smithi130 ceph-mon[129405]: pgmap v484: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:35 smithi040 ceph-mon[97954]: pgmap v484: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:35 smithi045 ceph-mon[128791]: pgmap v484: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:37.631 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:27:37.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 -- 172.21.15.40:0/345777641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 msgr2=0x7f2114105db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.851 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 --2- 172.21.15.40:0/345777641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114105db0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f21100098f0 tx=0x7f2110009c00 comp rx=0 tx=0).stop 2023-07-14T11:27:37.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 -- 172.21.15.40:0/345777641 shutdown_connections 2023-07-14T11:27:37.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 --2- 172.21.15.40:0/345777641 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211410c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 --2- 172.21.15.40:0/345777641 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114105db0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 --2- 172.21.15.40:0/345777641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f21140ffe40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.852 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 -- 172.21.15.40:0/345777641 >> 172.21.15.40:0/345777641 conn(0x7f21140fb4b0 msgr2=0x7f21140fd8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 -- 172.21.15.40:0/345777641 shutdown_connections 2023-07-14T11:27:37.853 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.847+0000 7f211a99d700 1 -- 172.21.15.40:0/345777641 wait complete. 2023-07-14T11:27:37.854 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.848+0000 7f211a99d700 1 Processor -- start 2023-07-14T11:27:37.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.848+0000 7f211a99d700 1 -- start start 2023-07-14T11:27:37.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f211419d9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211419df30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.855 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f21141980d0 con 0x7f21140ff9d0 2023-07-14T11:27:37.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2114198240 con 0x7f2114100380 2023-07-14T11:27:37.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a99d700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2114198400 con 0x7f21141059e0 2023-07-14T11:27:37.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.856 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211919a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211419df30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211999b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f211419d9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37324/0 (socket says 172.21.15.40:37324) 2023-07-14T11:27:37.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 -- 172.21.15.40:0/2915989434 learned_addr learned my addr 172.21.15.40:0/2915989434 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:37.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211919a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211419df30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35860/0 (socket says 172.21.15.40:35860) 2023-07-14T11:27:37.857 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 -- 172.21.15.40:0/2915989434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 msgr2=0x7f211419df30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211419df30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 -- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 msgr2=0x7f211419d9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f211419d9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211a19c700 1 -- 172.21.15.40:0/2915989434 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f21040095f0 con 0x7f21141059e0 2023-07-14T11:27:37.858 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.849+0000 7f211999b700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f211419d9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:27:37.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f211a19c700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f210c00ef30 tx=0x7f210c00c3a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f210affd700 1 -- 172.21.15.40:0/2915989434 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f210c018070 con 0x7f21141059e0 2023-07-14T11:27:37.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f210affd700 1 -- 172.21.15.40:0/2915989434 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f210c00f040 con 0x7f21141059e0 2023-07-14T11:27:37.859 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2110009520 con 0x7f21141059e0 2023-07-14T11:27:37.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f210affd700 1 -- 172.21.15.40:0/2915989434 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f210c00cd00 con 0x7f21141059e0 2023-07-14T11:27:37.860 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.850+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21141a4800 con 0x7f21141059e0 2023-07-14T11:27:37.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.855+0000 7f210affd700 1 -- 172.21.15.40:0/2915989434 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f210c003a00 con 0x7f21141059e0 2023-07-14T11:27:37.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 Processor -- start 2023-07-14T11:27:37.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 -- start start 2023-07-14T11:27:37.862 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc938c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bca0110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bb98cb0 con 0x56293bc934f0 2023-07-14T11:27:37.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bbeba40 con 0x56293bc9dd30 2023-07-14T11:27:37.863 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bbe4ed0 con 0x56293bc93e00 2023-07-14T11:27:37.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.856+0000 7f210affd700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 0x7f21000a3890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37326/0 (socket says 172.21.15.40:37326) 2023-07-14T11:27:37.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 -- 172.21.15.40:0/7794115 learned_addr learned my addr 172.21.15.40:0/7794115 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:37.864 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211919a700 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc938c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f210affd700 1 -- 172.21.15.40:0/2915989434 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f210c0ccb20 con 0x7f21141059e0 2023-07-14T11:27:37.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211919a700 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bca0110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211a19c700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 0x7f21000a3890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 -- 172.21.15.40:0/7794115 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 msgr2=0x56293bca0110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.865 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bca0110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 -- 172.21.15.40:0/7794115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 msgr2=0x56293bc938c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc938c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 -- 172.21.15.40:0/7794115 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56293ba12de0 con 0x56293bc93e00 2023-07-14T11:27:37.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.857+0000 7f211999b700 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f2110005e90 tx=0x7f2110005dc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=3a45ad287fc08647 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.866 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f211a19c700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 0x7f21000a3890 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f210c0c9ee0 tx=0x7f210c004dc0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f20fffff700 1 -- 172.21.15.40:0/7794115 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f211001c050 con 0x56293bc93e00 2023-07-14T11:27:37.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f20fffff700 1 -- 172.21.15.40:0/7794115 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f211001d040 con 0x56293bc93e00 2023-07-14T11:27:37.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f20fffff700 1 -- 172.21.15.40:0/7794115 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2110005430 con 0x56293bc93e00 2023-07-14T11:27:37.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 -- 172.21.15.40:0/7794115 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 msgr2=0x56293bc9d7f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.867 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7f2110005e90 tx=0x7f2110005dc0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 -- 172.21.15.40:0/7794115 shutdown_connections 2023-07-14T11:27:37.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bca0110 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc938c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/7794115 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9d7f0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.868 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 -- 172.21.15.40:0/7794115 >> 172.21.15.40:0/7794115 conn(0x56293bc88ff0 msgr2=0x56293bc8e930 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.858+0000 7f212f5c7b80 1 -- 172.21.15.40:0/7794115 shutdown_connections 2023-07-14T11:27:37.869 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.859+0000 7f212f5c7b80 1 -- 172.21.15.40:0/7794115 wait complete. 2023-07-14T11:27:37.871 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.859+0000 7f212f5c7b80 1 -- start start 2023-07-14T11:27:37.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.859+0000 7f212f5c7b80 1 Processor -- start 2023-07-14T11:27:37.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc9d6e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bc9af90 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.872 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bb84ac0 con 0x56293bc934f0 2023-07-14T11:27:37.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bb847c0 con 0x56293bc9dd30 2023-07-14T11:27:37.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293ba68920 con 0x56293bc93e00 2023-07-14T11:27:37.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f212f5c7b80 10 client.0 _ll_register_callbacks cb 0x56293bb82760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:27:37.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.873 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37334/0 (socket says 172.21.15.40:37334) 2023-07-14T11:27:37.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 -- 172.21.15.40:0/43368053 learned_addr learned my addr 172.21.15.40:0/43368053 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:37.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211919a700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc9d6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211919a700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bc9af90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 -- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 msgr2=0x56293bc9af90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.874 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bc9af90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 -- 172.21.15.40:0/43368053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 msgr2=0x56293bc9d6e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc9d6e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.860+0000 7f211999b700 1 -- 172.21.15.40:0/43368053 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f211410d110 con 0x56293bc93e00 2023-07-14T11:27:37.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f211999b700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f2110009ff0 tx=0x7f211000b790 comp rx=0 tx=0).ready entity=mon.1 client_cookie=c19ffc24d648e726 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.875 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f20fd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:27:37.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f211001c040 con 0x56293bc93e00 2023-07-14T11:27:37.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f2110005240 con 0x56293bc93e00 2023-07-14T11:27:37.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x56293bb3e4f0 con 0x56293bc93e00 2023-07-14T11:27:37.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2110005400 con 0x56293bc93e00 2023-07-14T11:27:37.876 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f212f5c7b80 10 client.26644 fetch_fsmap learned FSMap version 252 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x56293b9fbc60 con 0x56293bc93e00 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.861+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x56293bb3e4f0 con 0x56293bc93e00 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f211002a040 con 0x56293bc93e00 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 10 client.26644 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 10 client.26644 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:27:37.877 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 20 client.26644 populate_metadata read hostname 'smithi040' 2023-07-14T11:27:37.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bcad1d0 0x56293bcaf5e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 4 client.26644 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:27:37.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x56293bbddeb0 con 0x56293bcad1d0 2023-07-14T11:27:37.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f211919a700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bcad1d0 0x56293bcaf5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.878 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f211919a700 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bcad1d0 0x56293bcaf5e0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=106df3ae837b0e72 server_cookie=40ced1fd913eb4da in_seq=0 out_seq=0 2023-07-14T11:27:37.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.862+0000 7f20fd7fa700 10 client.26644 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:37.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.863+0000 7f20fd7fa700 1 -- 172.21.15.40:0/43368053 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x56293bbddeb0 con 0x56293bcad1d0 2023-07-14T11:27:37.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.863+0000 7f20fd7fa700 10 client.26644 handle_command_reply: tid=0 2023-07-14T11:27:37.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.863+0000 7f212f5c7b80 1 client.26644 shutdown 2023-07-14T11:27:37.879 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.863+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 msgr2=0x56293bc9a960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7f2110009ff0 tx=0x7f211000b790 comp rx=0 tx=0).stop 2023-07-14T11:27:37.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 shutdown_connections 2023-07-14T11:27:37.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bcad1d0 0x56293bcaf5e0 crc :-1 s=READY pgs=131 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc9dd30 0x56293bc9af90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.880 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc934f0 0x56293bc9d6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/43368053 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc93e00 0x56293bc9a960 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f20fd7fa700 0 client.26644 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:37.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 >> 172.21.15.40:0/43368053 conn(0x56293bc88ff0 msgr2=0x56293bc8e520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 shutdown_connections 2023-07-14T11:27:37.881 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 1 -- 172.21.15.40:0/43368053 wait complete. 2023-07-14T11:27:37.882 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.864+0000 7f212f5c7b80 20 client.26644 trim_cache size 0 max 16384 2023-07-14T11:27:37.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.881+0000 7f212f5c7b80 1 Processor -- start 2023-07-14T11:27:37.885 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.881+0000 7f212f5c7b80 1 -- start start 2023-07-14T11:27:37.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 0x56293bc9d280 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.886 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc9dd30 0x56293bc9a660 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bcaa0a0 con 0x56293bc9dd30 2023-07-14T11:27:37.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bc86140 con 0x56293bc93e00 2023-07-14T11:27:37.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bcafc40 con 0x56293bc934f0 2023-07-14T11:27:37.887 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211919a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 0x56293bc9d280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211919a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc9dd30 0x56293bc9a660 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35896/0 (socket says 172.21.15.40:35896) 2023-07-14T11:27:37.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211919a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 0x56293bc9d280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:37350/0 (socket says 172.21.15.40:37350) 2023-07-14T11:27:37.888 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 -- 172.21.15.40:0/2117317835 learned_addr learned my addr 172.21.15.40:0/2117317835 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 -- 172.21.15.40:0/2117317835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 msgr2=0x56293bc9d280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 0x56293bc9d280 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 -- 172.21.15.40:0/2117317835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc9dd30 msgr2=0x56293bc9a660 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc9dd30 0x56293bc9a660 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.889 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.882+0000 7f211999b700 1 -- 172.21.15.40:0/2117317835 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56293ba39250 con 0x56293bc93e00 2023-07-14T11:27:37.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f211999b700 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f2110009810 tx=0x7f2110005a40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=9f16b5031b338c3e server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f2108bff700 1 -- 172.21.15.40:0/2117317835 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f211000f980 con 0x56293bc93e00 2023-07-14T11:27:37.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f2108bff700 1 -- 172.21.15.40:0/2117317835 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f211000b8b0 con 0x56293bc93e00 2023-07-14T11:27:37.890 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f2108bff700 1 -- 172.21.15.40:0/2117317835 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2110005690 con 0x56293bc93e00 2023-07-14T11:27:37.893 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f212f5c7b80 1 -- 172.21.15.40:0/2117317835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 msgr2=0x56293bc9a120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.883+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7f2110009810 tx=0x7f2110005a40 comp rx=0 tx=0).stop 2023-07-14T11:27:37.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 -- 172.21.15.40:0/2117317835 shutdown_connections 2023-07-14T11:27:37.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc9dd30 0x56293bc9a660 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc934f0 0x56293bc9d280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.894 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/2117317835 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc93e00 0x56293bc9a120 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 -- 172.21.15.40:0/2117317835 >> 172.21.15.40:0/2117317835 conn(0x56293bc88ff0 msgr2=0x56293bc8d060 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 -- 172.21.15.40:0/2117317835 shutdown_connections 2023-07-14T11:27:37.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 -- 172.21.15.40:0/2117317835 wait complete. 2023-07-14T11:27:37.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 -- start start 2023-07-14T11:27:37.895 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.884+0000 7f212f5c7b80 1 Processor -- start 2023-07-14T11:27:37.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc93e00 0x56293bca4790 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc9dd30 0x56293bca2630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.896 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bca2d20 con 0x56293bc93e00 2023-07-14T11:27:37.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bca2e90 con 0x56293bc934f0 2023-07-14T11:27:37.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56293bca3050 con 0x56293bc9dd30 2023-07-14T11:27:37.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f212f5c7b80 10 client.0 _ll_register_callbacks cb 0x7f2110009740 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:27:37.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.897 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:35908/0 (socket says 172.21.15.40:35908) 2023-07-14T11:27:37.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 -- 172.21.15.40:0/1369341304 learned_addr learned my addr 172.21.15.40:0/1369341304 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:27:37.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc9dd30 0x56293bca2630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211999b700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc93e00 0x56293bca4790 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 -- 172.21.15.40:0/1369341304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc9dd30 msgr2=0x56293bca2630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc9dd30 0x56293bca2630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 -- 172.21.15.40:0/1369341304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc93e00 msgr2=0x56293bca4790 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:27:37.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc93e00 0x56293bca4790 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.885+0000 7f211919a700 1 -- 172.21.15.40:0/1369341304 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x56293b9fbc60 con 0x56293bc934f0 2023-07-14T11:27:37.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f211919a700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f2104000c00 tx=0x7f2104009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=f1c1ae217e1b444d server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:27:37.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f20fd7fa700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:27:37.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2104016040 con 0x56293bc934f0 2023-07-14T11:27:37.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f211001d040 con 0x56293bc934f0 2023-07-14T11:27:37.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f21040109e0 con 0x56293bc934f0 2023-07-14T11:27:37.900 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f2104010ba0 con 0x56293bc934f0 2023-07-14T11:27:37.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f212f5c7b80 10 client.26684 fetch_fsmap learned FSMap version 252 2023-07-14T11:27:37.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x56293ba112b0 con 0x56293bc934f0 2023-07-14T11:27:37.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.886+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f211001d040 con 0x56293bc934f0 2023-07-14T11:27:37.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f2104021340 con 0x56293bc934f0 2023-07-14T11:27:37.901 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 10 client.26684 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:27:37.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 10 client.26684 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:27:37.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 20 client.26684 populate_metadata read hostname 'smithi040' 2023-07-14T11:27:37.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bd021e0 0x56293bd045f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:27:37.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 4 client.26684 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:27:37.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x56293bba0050 con 0x56293bd021e0 2023-07-14T11:27:37.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f211999b700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bd021e0 0x56293bd045f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:27:37.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f211999b700 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bd021e0 0x56293bd045f0 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=1c1677ff1de6ccd2 server_cookie=c304484110397e51 in_seq=0 out_seq=0 2023-07-14T11:27:37.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.887+0000 7f20fd7fa700 10 client.26684 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:37.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f20fd7fa700 1 -- 172.21.15.40:0/1369341304 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x56293bba0050 con 0x56293bd021e0 2023-07-14T11:27:37.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f20fd7fa700 10 client.26684 handle_command_reply: tid=0 2023-07-14T11:27:37.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 client.26684 shutdown 2023-07-14T11:27:37.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 msgr2=0x56293bca4250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 crypto rx=0x7f2104000c00 tx=0x7f2104009310 comp rx=0 tx=0).stop 2023-07-14T11:27:37.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 shutdown_connections 2023-07-14T11:27:37.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56293bd021e0 0x56293bd045f0 crc :-1 s=READY pgs=132 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x56293bc9dd30 0x56293bca2630 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x56293bc934f0 0x56293bca4250 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.905 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f20fd7fa700 0 client.26684 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:27:37.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 --2- 172.21.15.40:0/1369341304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x56293bc93e00 0x56293bca4790 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 >> 172.21.15.40:0/1369341304 conn(0x56293bc88ff0 msgr2=0x56293bc92440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 shutdown_connections 2023-07-14T11:27:37.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 1 -- 172.21.15.40:0/1369341304 wait complete. 2023-07-14T11:27:37.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.888+0000 7f212f5c7b80 20 client.26684 trim_cache size 0 max 16384 2023-07-14T11:27:37.907 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:27:37.907 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:27:37.907 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:27:37.908 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:27:37.908 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:27:37.908 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:27:37.908 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:27:37.908 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:27:37.909 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:27:37.909 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:27:37.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 msgr2=0x7f21000a3890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 0x7f21000a3890 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f210c0c9ee0 tx=0x7f210c004dc0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 msgr2=0x7f2114197a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:27:37.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7f210c00ef30 tx=0x7f210c00c3a0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 shutdown_connections 2023-07-14T11:27:37.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f21000a13e0 0x7f21000a3890 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f21141059e0 0x7f2114197a60 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21140ff9d0 0x7f211419d9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 --2- 172.21.15.40:0/2915989434 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f2114100380 0x7f211419df30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:27:37.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 >> 172.21.15.40:0/2915989434 conn(0x7f21140fb4b0 msgr2=0x7f2114072740 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:27:37.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 shutdown_connections 2023-07-14T11:27:37.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:27:37.889+0000 7f211a99d700 1 -- 172.21.15.40:0/2915989434 wait complete. 2023-07-14T11:27:37.913 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:27:38.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:37 smithi040 ceph-mon[97954]: pgmap v485: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:37 smithi130 ceph-mon[129405]: pgmap v485: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:38.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:37 smithi045 ceph-mon[128791]: pgmap v485: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:39 smithi130 ceph-mon[129405]: pgmap v486: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:39 smithi040 ceph-mon[97954]: pgmap v486: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:40.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:39 smithi045 ceph-mon[128791]: pgmap v486: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:42.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:41 smithi130 ceph-mon[129405]: pgmap v487: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:42.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:41 smithi040 ceph-mon[97954]: pgmap v487: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:42.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:41 smithi045 ceph-mon[128791]: pgmap v487: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:44.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:43 smithi130 ceph-mon[129405]: pgmap v488: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:43 smithi040 ceph-mon[97954]: pgmap v488: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:44.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:43 smithi045 ceph-mon[128791]: pgmap v488: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:45 smithi130 ceph-mon[129405]: pgmap v489: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:45 smithi040 ceph-mon[97954]: pgmap v489: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:45 smithi045 ceph-mon[128791]: pgmap v489: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:48.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:47 smithi130 ceph-mon[129405]: pgmap v490: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:48.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:47 smithi040 ceph-mon[97954]: pgmap v490: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:48.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:47 smithi045 ceph-mon[128791]: pgmap v490: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:50.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:49 smithi130 ceph-mon[129405]: pgmap v491: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:49 smithi040 ceph-mon[97954]: pgmap v491: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:50.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:49 smithi045 ceph-mon[128791]: pgmap v491: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:51 smithi130 ceph-mon[129405]: pgmap v492: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:51 smithi040 ceph-mon[97954]: pgmap v492: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:51 smithi045 ceph-mon[128791]: pgmap v492: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:54.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:53 smithi130 ceph-mon[129405]: pgmap v493: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:54.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:53 smithi040 ceph-mon[97954]: pgmap v493: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:54.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:53 smithi045 ceph-mon[128791]: pgmap v493: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:55 smithi130 ceph-mon[129405]: pgmap v494: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:55 smithi040 ceph-mon[97954]: pgmap v494: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:56.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:55 smithi045 ceph-mon[128791]: pgmap v494: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:27:58.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:57 smithi130 ceph-mon[129405]: pgmap v495: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:58.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:57 smithi040 ceph-mon[97954]: pgmap v495: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:27:58.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:57 smithi045 ceph-mon[128791]: pgmap v495: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:27:59 smithi130 ceph-mon[129405]: pgmap v496: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:27:59 smithi040 ceph-mon[97954]: pgmap v496: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:00.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:27:59 smithi045 ceph-mon[128791]: pgmap v496: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:01 smithi130 ceph-mon[129405]: pgmap v497: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:02.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:01 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:28:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:01 smithi040 ceph-mon[97954]: pgmap v497: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:01 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:28:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:01 smithi045 ceph-mon[128791]: pgmap v497: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:01 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:28:03.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:02 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:28:03.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:02 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:28:03.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:02 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:28:03.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:02 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:28:03.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:02 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:28:03.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:02 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:28:03.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:02 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:28:03.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:02 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:28:03.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:02 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:28:04.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:03 smithi130 ceph-mon[129405]: pgmap v498: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:03 smithi040 ceph-mon[97954]: pgmap v498: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:04.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:03 smithi045 ceph-mon[128791]: pgmap v498: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:05 smithi130 ceph-mon[129405]: pgmap v499: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:28:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:05 smithi040 ceph-mon[97954]: pgmap v499: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:28:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:05 smithi045 ceph-mon[128791]: pgmap v499: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:28:07.914 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:28:08.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 -- 172.21.15.40:0/141276963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f37400ffae0 msgr2=0x7f37400fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 --2- 172.21.15.40:0/141276963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f37400ffae0 0x7f37400fff50 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 crypto rx=0x7f3734009990 tx=0x7f3734009ca0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 -- 172.21.15.40:0/141276963 shutdown_connections 2023-07-14T11:28:08.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 --2- 172.21.15.40:0/141276963 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374010c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 --2- 172.21.15.40:0/141276963 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3740105af0 0x7f3740105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 --2- 172.21.15.40:0/141276963 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f37400ffae0 0x7f37400fff50 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 -- 172.21.15.40:0/141276963 >> 172.21.15.40:0/141276963 conn(0x7f37400fb5e0 msgr2=0x7f37400fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 -- 172.21.15.40:0/141276963 shutdown_connections 2023-07-14T11:28:08.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.132+0000 7f37463ff700 1 -- 172.21.15.40:0/141276963 wait complete. 2023-07-14T11:28:08.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.133+0000 7f37463ff700 1 Processor -- start 2023-07-14T11:28:08.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 -- start start 2023-07-14T11:28:08.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374019bee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3740105af0 0x7f3740195a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3740196080 con 0x7f3740100490 2023-07-14T11:28:08.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37401961f0 con 0x7f3740105af0 2023-07-14T11:28:08.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f37463ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f37401963b0 con 0x7f37400ffae0 2023-07-14T11:28:08.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f373ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f373f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374019bee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f373ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:41866/0 (socket says 172.21.15.40:41866) 2023-07-14T11:28:08.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f373ffff700 1 -- 172.21.15.40:0/1267654384 learned_addr learned my addr 172.21.15.40:0/1267654384 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:08.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.134+0000 7f374499c700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3740105af0 0x7f3740195a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 -- 172.21.15.40:0/1267654384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3740105af0 msgr2=0x7f3740195a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3740105af0 0x7f3740195a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 -- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 msgr2=0x7f374019bee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374019bee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 -- 172.21.15.40:0/1267654384 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3738009c40 con 0x7f37400ffae0 2023-07-14T11:28:08.142 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373f7fe700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374019bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:28:08.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373ffff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f373000eb30 tx=0x7f373000ee40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373d7fa700 1 -- 172.21.15.40:0/1267654384 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3730009880 con 0x7f37400ffae0 2023-07-14T11:28:08.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3730009520 con 0x7f37400ffae0 2023-07-14T11:28:08.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373d7fa700 1 -- 172.21.15.40:0/1267654384 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f37300044e0 con 0x7f37400ffae0 2023-07-14T11:28:08.143 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.135+0000 7f373d7fa700 1 -- 172.21.15.40:0/1267654384 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3730003d80 con 0x7f37400ffae0 2023-07-14T11:28:08.144 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.136+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37340095f0 con 0x7f37400ffae0 2023-07-14T11:28:08.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.141+0000 7f373d7fa700 1 -- 172.21.15.40:0/1267654384 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3730003ee0 con 0x7f37400ffae0 2023-07-14T11:28:08.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.141+0000 7f3759c8cb80 1 Processor -- start 2023-07-14T11:28:08.145 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.141+0000 7f3759c8cb80 1 -- start start 2023-07-14T11:28:08.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f0502d0 0x558f4f0506a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05f9a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f0622c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4ef56cb0 con 0x558f4f0502d0 2023-07-14T11:28:08.146 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4efa9a40 con 0x558f4f050be0 2023-07-14T11:28:08.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4efa2ed0 con 0x558f4f05fee0 2023-07-14T11:28:08.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05f9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.147 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05f9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36090/0 (socket says 172.21.15.40:36090) 2023-07-14T11:28:08.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f373f7fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f0622c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f373ffff700 1 -- 172.21.15.40:0/2969706867 learned_addr learned my addr 172.21.15.40:0/2969706867 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:08.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f374499c700 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f0502d0 0x558f4f0506a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.142+0000 7f373d7fa700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 0x7f37280a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.148 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 -- 172.21.15.40:0/2969706867 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 msgr2=0x558f4f05f9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05f9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 -- 172.21.15.40:0/2969706867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f0502d0 msgr2=0x558f4f0506a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f0502d0 0x558f4f0506a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 -- 172.21.15.40:0/2969706867 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558f4edd0de0 con 0x558f4f05fee0 2023-07-14T11:28:08.149 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f374499c700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 0x7f37280a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.150 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373d7fa700 1 -- 172.21.15.40:0/1267654384 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3730014070 con 0x7f37400ffae0 2023-07-14T11:28:08.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f0622c0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f37340059d0 tx=0x7f373400b610 comp rx=0 tx=0).ready entity=mon.1 client_cookie=acb28e84ca8dbe22 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f37267fc700 1 -- 172.21.15.40:0/2969706867 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f373401d070 con 0x558f4f05fee0 2023-07-14T11:28:08.153 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f37267fc700 1 -- 172.21.15.40:0/2969706867 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f373401e040 con 0x558f4f05fee0 2023-07-14T11:28:08.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.143+0000 7f37267fc700 1 -- 172.21.15.40:0/2969706867 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3734004cf0 con 0x558f4f05fee0 2023-07-14T11:28:08.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2969706867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 msgr2=0x558f4f0622c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f374499c700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 0x7f37280a3770 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f3738004a60 tx=0x7f3738009040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f0622c0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f37340059d0 tx=0x7f373400b610 comp rx=0 tx=0).stop 2023-07-14T11:28:08.154 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2969706867 shutdown_connections 2023-07-14T11:28:08.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f0622c0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f0502d0 0x558f4f0506a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2969706867 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05f9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2969706867 >> 172.21.15.40:0/2969706867 conn(0x558f4f046da0 msgr2=0x558f4f04c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2969706867 shutdown_connections 2023-07-14T11:28:08.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.144+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2969706867 wait complete. 2023-07-14T11:28:08.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 -- start start 2023-07-14T11:28:08.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 Processor -- start 2023-07-14T11:28:08.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05cb10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f05fee0 0x558f4f05d140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4ef42ac0 con 0x558f4f05fee0 2023-07-14T11:28:08.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4ef427c0 con 0x558f4f050be0 2023-07-14T11:28:08.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4ee26920 con 0x558f4f0502d0 2023-07-14T11:28:08.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.145+0000 7f3759c8cb80 10 client.0 _ll_register_callbacks cb 0x558f4ef40760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:28:08.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:41884/0 (socket says 172.21.15.40:41884) 2023-07-14T11:28:08.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 -- 172.21.15.40:0/1712664870 learned_addr learned my addr 172.21.15.40:0/1712664870 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:08.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f374499c700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f05fee0 0x558f4f05d140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373f7fe700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05cb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 -- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 msgr2=0x558f4f05cb10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05cb10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 -- 172.21.15.40:0/1712664870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f05fee0 msgr2=0x558f4f05d140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f05fee0 0x558f4f05d140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 -- 172.21.15.40:0/1712664870 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f374010d220 con 0x558f4f0502d0 2023-07-14T11:28:08.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f373ffff700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f373000eb00 tx=0x7f3730004750 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e24fe9f24fd4303a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f371f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:28:08.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.146+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3730099f40 con 0x558f4f0502d0 2023-07-14T11:28:08.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f373000cbd0 con 0x558f4f0502d0 2023-07-14T11:28:08.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x558f4eefc4f0 con 0x558f4f0502d0 2023-07-14T11:28:08.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3730099cb0 con 0x558f4f0502d0 2023-07-14T11:28:08.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 10 client.26662 fetch_fsmap learned FSMap version 252 2023-07-14T11:28:08.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x558f4edb9c60 con 0x558f4f0502d0 2023-07-14T11:28:08.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x558f4eefc4f0 con 0x558f4f0502d0 2023-07-14T11:28:08.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f37300cd120 con 0x558f4f0502d0 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 10 client.26662 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 10 client.26662 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 20 client.26662 populate_metadata read hostname 'smithi040' 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f06f750 0x558f4f071b30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 4 client.26662 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:28:08.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x558f4ef9beb0 con 0x558f4f06f750 2023-07-14T11:28:08.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.147+0000 7f373f7fe700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f06f750 0x558f4f071b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.148+0000 7f373f7fe700 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f06f750 0x558f4f071b30 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3083b4a06b810cf5 server_cookie=2d22c32a55ebdb62 in_seq=0 out_seq=0 2023-07-14T11:28:08.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.148+0000 7f371f7fe700 10 client.26662 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:08.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f371f7fe700 1 -- 172.21.15.40:0/1712664870 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x558f4ef9beb0 con 0x558f4f06f750 2023-07-14T11:28:08.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f371f7fe700 10 client.26662 handle_command_reply: tid=0 2023-07-14T11:28:08.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 client.26662 shutdown 2023-07-14T11:28:08.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 msgr2=0x558f4f05f900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f373000eb00 tx=0x7f3730004750 comp rx=0 tx=0).stop 2023-07-14T11:28:08.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 shutdown_connections 2023-07-14T11:28:08.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f06f750 0x558f4f071b30 crc :-1 s=READY pgs=133 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f05fee0 0x558f4f05d140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f0502d0 0x558f4f05f900 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1712664870 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f050be0 0x558f4f05cb10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f371f7fe700 0 client.26662 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:08.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 >> 172.21.15.40:0/1712664870 conn(0x558f4f046da0 msgr2=0x558f4f04c290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 shutdown_connections 2023-07-14T11:28:08.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1712664870 wait complete. 2023-07-14T11:28:08.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.149+0000 7f3759c8cb80 20 client.26662 trim_cache size 0 max 16384 2023-07-14T11:28:08.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 Processor -- start 2023-07-14T11:28:08.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 -- start start 2023-07-14T11:28:08.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f05c320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f05c860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.171 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f046970 con 0x558f4f050be0 2023-07-14T11:28:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f0521a0 con 0x558f4f0502d0 2023-07-14T11:28:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f05b0c0 con 0x558f4f05fee0 2023-07-14T11:28:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36106/0 (socket says 172.21.15.40:36106) 2023-07-14T11:28:08.172 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f373ffff700 1 -- 172.21.15.40:0/1233482108 learned_addr learned my addr 172.21.15.40:0/1233482108 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:08.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.167+0000 7f374499c700 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f05c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373f7fe700 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f05c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 -- 172.21.15.40:0/1233482108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 msgr2=0x558f4f05c860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f05c860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.173 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 -- 172.21.15.40:0/1233482108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 msgr2=0x558f4f05c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f05c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 -- 172.21.15.40:0/1233482108 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x558f4edf7250 con 0x558f4f0502d0 2023-07-14T11:28:08.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f373ffff700 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f3730096670 tx=0x7f37300cd590 comp rx=0 tx=0).ready entity=mon.2 client_cookie=1b06261a7546f088 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f3726ffd700 1 -- 172.21.15.40:0/1233482108 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3730099f40 con 0x558f4f0502d0 2023-07-14T11:28:08.174 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f3726ffd700 1 -- 172.21.15.40:0/1233482108 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3730099cb0 con 0x558f4f0502d0 2023-07-14T11:28:08.175 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.168+0000 7f3726ffd700 1 -- 172.21.15.40:0/1233482108 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f373000cbd0 con 0x558f4f0502d0 2023-07-14T11:28:08.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1233482108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 msgr2=0x558f4f05f430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 crypto rx=0x7f3730096670 tx=0x7f37300cd590 comp rx=0 tx=0).stop 2023-07-14T11:28:08.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1233482108 shutdown_connections 2023-07-14T11:28:08.178 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f05c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f05f430 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/1233482108 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f05c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1233482108 >> 172.21.15.40:0/1233482108 conn(0x558f4f046da0 msgr2=0x558f4f057e60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1233482108 shutdown_connections 2023-07-14T11:28:08.179 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- 172.21.15.40:0/1233482108 wait complete. 2023-07-14T11:28:08.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.169+0000 7f3759c8cb80 1 -- start start 2023-07-14T11:28:08.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.170+0000 7f3759c8cb80 1 Processor -- start 2023-07-14T11:28:08.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.170+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f0654f0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.180 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f065a30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f0c35c0 con 0x558f4f050be0 2023-07-14T11:28:08.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f0c3700 con 0x558f4f0502d0 2023-07-14T11:28:08.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x558f4f0c38c0 con 0x558f4f05fee0 2023-07-14T11:28:08.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f3759c8cb80 10 client.0 _ll_register_callbacks cb 0x558f4ef40760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:28:08.181 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36114/0 (socket says 172.21.15.40:36114) 2023-07-14T11:28:08.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f0654f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 -- 172.21.15.40:0/2369784376 learned_addr learned my addr 172.21.15.40:0/2369784376 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:08.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f374499c700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f065a30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.182 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 -- 172.21.15.40:0/2369784376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 msgr2=0x558f4f065a30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f065a30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 -- 172.21.15.40:0/2369784376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 msgr2=0x558f4f0654f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:08.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f0654f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 -- 172.21.15.40:0/2369784376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558f4edb9c60 con 0x558f4f0502d0 2023-07-14T11:28:08.183 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f0654f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:28:08.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f373ffff700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f3730009230 tx=0x7f37300cad50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=b7cd8de2d108f658 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:08.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.171+0000 7f371f7fe700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:28:08.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3730099f40 con 0x558f4f0502d0 2023-07-14T11:28:08.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(111..111 src has 1..111) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f373000cbd0 con 0x558f4f0502d0 2023-07-14T11:28:08.184 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f373401e040 con 0x558f4f0502d0 2023-07-14T11:28:08.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f373000cdd0 con 0x558f4f0502d0 2023-07-14T11:28:08.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 10 client.26702 fetch_fsmap learned FSMap version 252 2023-07-14T11:28:08.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x558f4edcf2b0 con 0x558f4f0502d0 2023-07-14T11:28:08.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f373401e040 con 0x558f4f0502d0 2023-07-14T11:28:08.185 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f373009a3d0 con 0x558f4f0502d0 2023-07-14T11:28:08.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 10 client.26702 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:28:08.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 10 client.26702 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:28:08.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 20 client.26702 populate_metadata read hostname 'smithi040' 2023-07-14T11:28:08.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f0c4b20 0x558f4f0c6f00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:08.186 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 4 client.26702 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.172+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x558f4ef5e050 con 0x558f4f0c4b20 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f0c4b20 0x558f4f0c6f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f373f7fe700 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f0c4b20 0x558f4f0c6f00 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=836f1f7edb3f286d server_cookie=13cc00e61d6e4a71 in_seq=0 out_seq=0 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f371f7fe700 10 client.26702 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f371f7fe700 1 -- 172.21.15.40:0/2369784376 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x558f4ef5e050 con 0x558f4f0c4b20 2023-07-14T11:28:08.187 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f371f7fe700 10 client.26702 handle_command_reply: tid=0 2023-07-14T11:28:08.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.173+0000 7f3759c8cb80 1 client.26702 shutdown 2023-07-14T11:28:08.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 msgr2=0x558f4f064fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 crypto rx=0x7f3730009230 tx=0x7f37300cad50 comp rx=0 tx=0).stop 2023-07-14T11:28:08.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 shutdown_connections 2023-07-14T11:28:08.188 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x558f4f0c4b20 0x558f4f0c6f00 crc :-1 s=READY pgs=134 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x558f4f05fee0 0x558f4f065a30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x558f4f0502d0 0x558f4f064fb0 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 --2- 172.21.15.40:0/2369784376 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x558f4f050be0 0x558f4f0654f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f371f7fe700 0 client.26702 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:08.189 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 >> 172.21.15.40:0/2369784376 conn(0x558f4f046da0 msgr2=0x558f4f0523e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 shutdown_connections 2023-07-14T11:28:08.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 1 -- 172.21.15.40:0/2369784376 wait complete. 2023-07-14T11:28:08.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.174+0000 7f3759c8cb80 20 client.26702 trim_cache size 0 max 16384 2023-07-14T11:28:08.190 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:28:08.191 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:28:08.191 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:28:08.191 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:28:08.191 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:28:08.192 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:28:08.192 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:28:08.192 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:28:08.192 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:28:08.192 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:28:08.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 msgr2=0x7f37280a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 0x7f37280a3770 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f3738004a60 tx=0x7f3738009040 comp rx=0 tx=0).stop 2023-07-14T11:28:08.193 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 msgr2=0x7f374019b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:08.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7f373000eb30 tx=0x7f373000ee40 comp rx=0 tx=0).stop 2023-07-14T11:28:08.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 shutdown_connections 2023-07-14T11:28:08.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f37280a12c0 0x7f37280a3770 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3740105af0 0x7f3740195a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f37400ffae0 0x7f374019b9a0 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 --2- 172.21.15.40:0/1267654384 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3740100490 0x7f374019bee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:08.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 >> 172.21.15.40:0/1267654384 conn(0x7f37400fb5e0 msgr2=0x7f37400fbbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:08.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 shutdown_connections 2023-07-14T11:28:08.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:08.175+0000 7f37463ff700 1 -- 172.21.15.40:0/1267654384 wait complete. 2023-07-14T11:28:08.196 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:07 smithi040 ceph-mon[97954]: pgmap v500: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:08.197 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:28:08.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:07 smithi130 ceph-mon[129405]: pgmap v500: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:07 smithi045 ceph-mon[128791]: pgmap v500: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:09 smithi130 ceph-mon[129405]: pgmap v501: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:10.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:09 smithi130 ceph-mon[129405]: osdmap e112: 12 total, 12 up, 12 in 2023-07-14T11:28:10.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:09 smithi040 ceph-mon[97954]: pgmap v501: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:10.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:09 smithi040 ceph-mon[97954]: osdmap e112: 12 total, 12 up, 12 in 2023-07-14T11:28:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:09 smithi045 ceph-mon[128791]: pgmap v501: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:28:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:09 smithi045 ceph-mon[128791]: osdmap e112: 12 total, 12 up, 12 in 2023-07-14T11:28:12.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:11 smithi130 ceph-mon[129405]: pgmap v503: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:12.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:11 smithi040 ceph-mon[97954]: pgmap v503: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:12.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:11 smithi045 ceph-mon[128791]: pgmap v503: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:14.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:13 smithi130 ceph-mon[129405]: pgmap v504: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:14.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:13 smithi040 ceph-mon[97954]: pgmap v504: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:14.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:13 smithi045 ceph-mon[128791]: pgmap v504: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:28:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:15 smithi130 ceph-mon[129405]: osdmap e113: 12 total, 12 up, 12 in 2023-07-14T11:28:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:15 smithi130 ceph-mon[129405]: pgmap v506: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:15 smithi040 ceph-mon[97954]: osdmap e113: 12 total, 12 up, 12 in 2023-07-14T11:28:16.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:15 smithi040 ceph-mon[97954]: pgmap v506: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:16.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:15 smithi045 ceph-mon[128791]: osdmap e113: 12 total, 12 up, 12 in 2023-07-14T11:28:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:15 smithi045 ceph-mon[128791]: pgmap v506: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:18.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:17 smithi130 ceph-mon[129405]: pgmap v507: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:18.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:17 smithi040 ceph-mon[97954]: pgmap v507: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:18.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:17 smithi045 ceph-mon[128791]: pgmap v507: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:28:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:19 smithi130 ceph-mon[129405]: pgmap v508: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:28:20.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:19 smithi040 ceph-mon[97954]: pgmap v508: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:28:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:19 smithi045 ceph-mon[128791]: pgmap v508: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:28:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:21 smithi130 ceph-mon[129405]: pgmap v509: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:22.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:21 smithi040 ceph-mon[97954]: pgmap v509: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:21 smithi045 ceph-mon[128791]: pgmap v509: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:23 smithi130 ceph-mon[129405]: pgmap v510: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:23 smithi040 ceph-mon[97954]: pgmap v510: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:24.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:23 smithi045 ceph-mon[128791]: pgmap v510: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:28:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:25 smithi130 ceph-mon[129405]: pgmap v511: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:28:26.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:25 smithi040 ceph-mon[97954]: pgmap v511: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:28:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:25 smithi045 ceph-mon[128791]: pgmap v511: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:28:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:27 smithi130 ceph-mon[129405]: pgmap v512: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:28.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:27 smithi040 ceph-mon[97954]: pgmap v512: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:28.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:27 smithi045 ceph-mon[128791]: pgmap v512: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:29 smithi130 ceph-mon[129405]: pgmap v513: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:29 smithi040 ceph-mon[97954]: pgmap v513: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:30.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:29 smithi045 ceph-mon[128791]: pgmap v513: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:32.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:31 smithi130 ceph-mon[129405]: pgmap v514: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:31 smithi040 ceph-mon[97954]: pgmap v514: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:32.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:31 smithi045 ceph-mon[128791]: pgmap v514: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:33 smithi130 ceph-mon[129405]: pgmap v515: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:33 smithi040 ceph-mon[97954]: pgmap v515: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:33 smithi045 ceph-mon[128791]: pgmap v515: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:35 smithi130 ceph-mon[129405]: pgmap v516: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:35 smithi040 ceph-mon[97954]: pgmap v516: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:35 smithi045 ceph-mon[128791]: pgmap v516: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:38.198 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:28:38.266 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:37 smithi040 ceph-mon[97954]: pgmap v517: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:37 smithi130 ceph-mon[129405]: pgmap v517: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:38.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:37 smithi045 ceph-mon[128791]: pgmap v517: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:38.417 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.413+0000 7f3db2fff700 1 -- 172.21.15.40:0/3687147824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 msgr2=0x7f3dac1068d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.413+0000 7f3db2fff700 1 --2- 172.21.15.40:0/3687147824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac1068d0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f3d980099c0 tx=0x7f3d98009cd0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 -- 172.21.15.40:0/3687147824 shutdown_connections 2023-07-14T11:28:38.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 --2- 172.21.15.40:0/3687147824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac10eb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 --2- 172.21.15.40:0/3687147824 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 0x7f3dac102340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 --2- 172.21.15.40:0/3687147824 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac1068d0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 -- 172.21.15.40:0/3687147824 >> 172.21.15.40:0/3687147824 conn(0x7f3dac0fd7e0 msgr2=0x7f3dac0ffbf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 -- 172.21.15.40:0/3687147824 shutdown_connections 2023-07-14T11:28:38.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.414+0000 7f3db2fff700 1 -- 172.21.15.40:0/3687147824 wait complete. 2023-07-14T11:28:38.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 Processor -- start 2023-07-14T11:28:38.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 -- start start 2023-07-14T11:28:38.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 0x7f3dac197e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac19c7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dac19ce20 con 0x7f3dac106fb0 2023-07-14T11:28:38.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dac19cf90 con 0x7f3dac101f70 2023-07-14T11:28:38.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db2fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3dac19d150 con 0x7f3dac102910 2023-07-14T11:28:38.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3dabfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3dabfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:48292/0 (socket says 172.21.15.40:48292) 2023-07-14T11:28:38.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 0x7f3dac197e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.415+0000 7f3dabfff700 1 -- 172.21.15.40:0/1002814319 learned_addr learned my addr 172.21.15.40:0/1002814319 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:38.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3db159c700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac19c7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 -- 172.21.15.40:0/1002814319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 msgr2=0x7f3dac197e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 0x7f3dac197e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 -- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 msgr2=0x7f3dac19c7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac19c7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 -- 172.21.15.40:0/1002814319 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d980095f0 con 0x7f3dac102910 2023-07-14T11:28:38.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3db159c700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac19c7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:28:38.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3dabfff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f3d980098e0 tx=0x7f3d9800f690 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3da9ffb700 1 -- 172.21.15.40:0/1002814319 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d9801d070 con 0x7f3dac102910 2023-07-14T11:28:38.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3da9ffb700 1 -- 172.21.15.40:0/1002814319 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d98005510 con 0x7f3dac102910 2023-07-14T11:28:38.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3da9ffb700 1 -- 172.21.15.40:0/1002814319 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d98004a00 con 0x7f3dac102910 2023-07-14T11:28:38.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3dac19d3d0 con 0x7f3dac102910 2023-07-14T11:28:38.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.416+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dac19d650 con 0x7f3dac102910 2023-07-14T11:28:38.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.420+0000 7f3da9ffb700 1 -- 172.21.15.40:0/1002814319 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3d98004bc0 con 0x7f3dac102910 2023-07-14T11:28:38.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.421+0000 7f3dc6880b80 1 Processor -- start 2023-07-14T11:28:38.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.421+0000 7f3da9ffb700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 0x7f3d9c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.421+0000 7f3da9ffb700 1 -- 172.21.15.40:0/1002814319 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3d980cdc60 con 0x7f3dac102910 2023-07-14T11:28:38.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.421+0000 7f3dc6880b80 1 -- start start 2023-07-14T11:28:38.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.421+0000 7f3db159c700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 0x7f3d9c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022cd6a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022d8ad0 0x5629022daeb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.429 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629021d3cb0 con 0x5629022cd2d0 2023-07-14T11:28:38.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x562902226a40 con 0x5629022cdbe0 2023-07-14T11:28:38.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x56290221fed0 con 0x5629022d8ad0 2023-07-14T11:28:38.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022cd6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.430 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41652/0 (socket says 172.21.15.40:41652) 2023-07-14T11:28:38.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 -- 172.21.15.40:0/2655611363 learned_addr learned my addr 172.21.15.40:0/2655611363 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:38.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3db159c700 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022d8ad0 0x5629022daeb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3db159c700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 0x7f3d9c0a3770 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f3da40093a0 tx=0x7f3da400b040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 -- 172.21.15.40:0/2655611363 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022d8ad0 msgr2=0x5629022daeb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.431 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022d8ad0 0x5629022daeb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 -- 172.21.15.40:0/2655611363 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 msgr2=0x5629022cd6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022cd6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.422+0000 7f3dabfff700 1 -- 172.21.15.40:0/2655611363 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x56290204dde0 con 0x5629022cdbe0 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3dabfff700 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f3d98096ed0 tx=0x7f3d9809a8b0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=63cabc711709fa45 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3d96ffd700 1 -- 172.21.15.40:0/2655611363 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d9809ea00 con 0x5629022cdbe0 2023-07-14T11:28:38.432 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3d96ffd700 1 -- 172.21.15.40:0/2655611363 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d980a2d90 con 0x5629022cdbe0 2023-07-14T11:28:38.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3d96ffd700 1 -- 172.21.15.40:0/2655611363 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d98005250 con 0x5629022cdbe0 2023-07-14T11:28:38.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3dc6880b80 1 -- 172.21.15.40:0/2655611363 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 msgr2=0x5629022d8590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.423+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 crypto rx=0x7f3d98096ed0 tx=0x7f3d9809a8b0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 -- 172.21.15.40:0/2655611363 shutdown_connections 2023-07-14T11:28:38.433 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022d8ad0 0x5629022daeb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022cd6a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/2655611363 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cdbe0 0x5629022d8590 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 -- 172.21.15.40:0/2655611363 >> 172.21.15.40:0/2655611363 conn(0x5629022c3da0 msgr2=0x5629022c96a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 -- 172.21.15.40:0/2655611363 shutdown_connections 2023-07-14T11:28:38.434 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 -- 172.21.15.40:0/2655611363 wait complete. 2023-07-14T11:28:38.435 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 -- start start 2023-07-14T11:28:38.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.424+0000 7f3dc6880b80 1 Processor -- start 2023-07-14T11:28:38.437 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022d93e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022d9920 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629021bfac0 con 0x5629022cd2d0 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629021bf7c0 con 0x5629022d8ad0 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629020a3920 con 0x5629022cdbe0 2023-07-14T11:28:38.438 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dc6880b80 10 client.0 _ll_register_callbacks cb 0x5629021bd760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:28:38.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46272/0 (socket says 172.21.15.40:46272) 2023-07-14T11:28:38.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3db0d9b700 1 -- 172.21.15.40:0/114236901 learned_addr learned my addr 172.21.15.40:0/114236901 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:38.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3dabfff700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022d93e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.439 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.425+0000 7f3db159c700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022d9920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 -- 172.21.15.40:0/114236901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 msgr2=0x5629022d93e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022d93e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 -- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 msgr2=0x5629022d9920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022d9920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.440 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 -- 172.21.15.40:0/114236901 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3dac19d3d0 con 0x5629022cd2d0 2023-07-14T11:28:38.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3dabfff700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022d93e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:28:38.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3da0012860 tx=0x7f3da0012b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1c015958491c7b76 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3d8bfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:28:38.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3da0017070 con 0x5629022cd2d0 2023-07-14T11:28:38.441 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x5629021794f0 con 0x5629022cd2d0 2023-07-14T11:28:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3da000f540 con 0x5629022cd2d0 2023-07-14T11:28:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3da00122e0 con 0x5629022cd2d0 2023-07-14T11:28:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3dc6880b80 10 client.16656 fetch_fsmap learned FSMap version 252 2023-07-14T11:28:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.426+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x562902036c60 con 0x5629022cd2d0 2023-07-14T11:28:38.442 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x5629021794f0 con 0x5629022cd2d0 2023-07-14T11:28:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3da0010840 con 0x5629022cd2d0 2023-07-14T11:28:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 10 client.16656 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:28:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 10 client.16656 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:28:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 20 client.16656 populate_metadata read hostname 'smithi040' 2023-07-14T11:28:38.443 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629022e8160 0x5629022ea570 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 4 client.16656 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:28:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x562902218eb0 con 0x5629022e8160 2023-07-14T11:28:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dabfff700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629022e8160 0x5629022ea570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.427+0000 7f3dabfff700 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629022e8160 0x5629022ea570 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=7d6c4683a963b70e server_cookie=3c428a72f8159f5a in_seq=0 out_seq=0 2023-07-14T11:28:38.444 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.428+0000 7f3d8bfff700 10 client.16656 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.428+0000 7f3d8bfff700 1 -- 172.21.15.40:0/114236901 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x562902218eb0 con 0x5629022e8160 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.428+0000 7f3d8bfff700 10 client.16656 handle_command_reply: tid=0 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.428+0000 7f3dc6880b80 1 client.16656 shutdown 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 msgr2=0x5629022d8ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 crypto rx=0x7f3da0012860 tx=0x7f3da0012b70 comp rx=0 tx=0).stop 2023-07-14T11:28:38.445 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 shutdown_connections 2023-07-14T11:28:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x5629022e8160 0x5629022ea570 crc :-1 s=READY pgs=135 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022d9920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022d8ea0 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3d8bfff700 0 client.16656 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:38.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/114236901 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022d93e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 >> 172.21.15.40:0/114236901 conn(0x5629022c3da0 msgr2=0x5629022c9290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 shutdown_connections 2023-07-14T11:28:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 1 -- 172.21.15.40:0/114236901 wait complete. 2023-07-14T11:28:38.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.429+0000 7f3dc6880b80 20 client.16656 trim_cache size 0 max 16384 2023-07-14T11:28:38.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 Processor -- start 2023-07-14T11:28:38.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 -- start start 2023-07-14T11:28:38.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022e4450 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 0x5629022e3680 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022d8180 con 0x5629022d8ad0 2023-07-14T11:28:38.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022e3fc0 con 0x5629022cd2d0 2023-07-14T11:28:38.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022d6090 con 0x5629022cdbe0 2023-07-14T11:28:38.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41678/0 (socket says 172.21.15.40:41678) 2023-07-14T11:28:38.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3db0d9b700 1 -- 172.21.15.40:0/1497990430 learned_addr learned my addr 172.21.15.40:0/1497990430 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:38.453 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3dabfff700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022e4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.447+0000 7f3db159c700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 0x5629022e3680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 -- 172.21.15.40:0/1497990430 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 msgr2=0x5629022e4450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022e4450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 -- 172.21.15.40:0/1497990430 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 msgr2=0x5629022e3680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.454 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 0x5629022e3680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 -- 172.21.15.40:0/1497990430 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x562902074250 con 0x5629022cd2d0 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db159c700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 0x5629022e3680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f3da0005a60 tx=0x7f3da00122e0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=a7fe677c3e81175a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3d977fe700 1 -- 172.21.15.40:0/1497990430 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3da0017070 con 0x5629022cd2d0 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3d977fe700 1 -- 172.21.15.40:0/1497990430 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3da0018040 con 0x5629022cd2d0 2023-07-14T11:28:38.455 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.448+0000 7f3d977fe700 1 -- 172.21.15.40:0/1497990430 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3da000f540 con 0x5629022cd2d0 2023-07-14T11:28:38.458 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- 172.21.15.40:0/1497990430 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 msgr2=0x5629022dced0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 crypto rx=0x7f3da0005a60 tx=0x7f3da00122e0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- 172.21.15.40:0/1497990430 shutdown_connections 2023-07-14T11:28:38.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022d8ad0 0x5629022e3680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022cd2d0 0x5629022dced0 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.459 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/1497990430 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022e4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- 172.21.15.40:0/1497990430 >> 172.21.15.40:0/1497990430 conn(0x5629022c3da0 msgr2=0x5629022d2440 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- 172.21.15.40:0/1497990430 shutdown_connections 2023-07-14T11:28:38.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- 172.21.15.40:0/1497990430 wait complete. 2023-07-14T11:28:38.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 -- start start 2023-07-14T11:28:38.460 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.449+0000 7f3dc6880b80 1 Processor -- start 2023-07-14T11:28:38.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022dc940 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022dce80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022db410 con 0x5629022cd2d0 2023-07-14T11:28:38.461 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022db580 con 0x5629022d8ad0 2023-07-14T11:28:38.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x5629022db740 con 0x5629022cdbe0 2023-07-14T11:28:38.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dc6880b80 10 client.0 _ll_register_callbacks cb 0x7f3da0009ed0 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:28:38.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022dc940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3db159c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.462 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3dabfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022dce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3db0d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022dc940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:46288/0 (socket says 172.21.15.40:46288) 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3db159c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:41688/0 (socket says 172.21.15.40:41688) 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.450+0000 7f3db0d9b700 1 -- 172.21.15.40:0/3722222863 learned_addr learned my addr 172.21.15.40:0/3722222863 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 -- 172.21.15.40:0/3722222863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 msgr2=0x5629022dce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022dce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.463 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 -- 172.21.15.40:0/3722222863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 msgr2=0x5629022dc940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:28:38.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022dc940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 -- 172.21.15.40:0/3722222863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x562902036c60 con 0x5629022d8ad0 2023-07-14T11:28:38.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3dabfff700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022dce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:28:38.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3db159c700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f3da4009c10 tx=0x7f3da4004ad0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=4f6bfb0fb6b0d90b server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:28:38.464 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3d8bfff700 10 client.0 ms_handle_connect on v2:172.21.15.45:3300/0 2023-07-14T11:28:38.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3da400e050 con 0x5629022d8ad0 2023-07-14T11:28:38.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f3d980a2d90 con 0x5629022d8ad0 2023-07-14T11:28:38.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mon.2 v2:172.21.15.45:3300/0 2 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3da4016a60 con 0x5629022d8ad0 2023-07-14T11:28:38.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f3da4016c20 con 0x5629022d8ad0 2023-07-14T11:28:38.465 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3dc6880b80 10 client.26720 fetch_fsmap learned FSMap version 252 2023-07-14T11:28:38.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.451+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x56290204c2b0 con 0x5629022d8ad0 2023-07-14T11:28:38.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mon.2 v2:172.21.15.45:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d980a2d90 con 0x5629022d8ad0 2023-07-14T11:28:38.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mon.2 v2:172.21.15.45:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f3da402b410 con 0x5629022d8ad0 2023-07-14T11:28:38.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 10 client.26720 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:28:38.466 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 10 client.26720 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:28:38.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 20 client.26720 populate_metadata read hostname 'smithi040' 2023-07-14T11:28:38.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56290233d130 0x56290233f540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:28:38.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 4 client.26720 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:28:38.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x5629021db050 con 0x56290233d130 2023-07-14T11:28:38.467 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.452+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56290233d130 0x56290233f540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:28:38.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3db0d9b700 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56290233d130 0x56290233f540 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=74ac81b1803d11bb server_cookie=6fe1aca952986081 in_seq=0 out_seq=0 2023-07-14T11:28:38.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3d8bfff700 10 client.26720 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:38.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3d8bfff700 1 -- 172.21.15.40:0/3722222863 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x5629021db050 con 0x56290233d130 2023-07-14T11:28:38.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3d8bfff700 10 client.26720 handle_command_reply: tid=0 2023-07-14T11:28:38.468 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 client.26720 shutdown 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 msgr2=0x5629022dad00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 crypto rx=0x7f3da4009c10 tx=0x7f3da4004ad0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 shutdown_connections 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x56290233d130 0x56290233f540 crc :-1 s=READY pgs=136 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x5629022d8ad0 0x5629022dad00 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.469 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3d8bfff700 0 client.26720 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:28:38.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x5629022cd2d0 0x5629022dc940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 --2- 172.21.15.40:0/3722222863 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x5629022cdbe0 0x5629022dce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 >> 172.21.15.40:0/3722222863 conn(0x5629022c3da0 msgr2=0x5629022e4450 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 shutdown_connections 2023-07-14T11:28:38.470 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.453+0000 7f3dc6880b80 1 -- 172.21.15.40:0/3722222863 wait complete. 2023-07-14T11:28:38.471 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3dc6880b80 20 client.26720 trim_cache size 0 max 16384 2023-07-14T11:28:38.471 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:28:38.471 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:28:38.471 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:28:38.472 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:28:38.472 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:28:38.472 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:28:38.472 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:28:38.472 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:28:38.473 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:28:38.473 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:28:38.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 msgr2=0x7f3d9c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 0x7f3d9c0a3770 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f3da40093a0 tx=0x7f3da400b040 comp rx=0 tx=0).stop 2023-07-14T11:28:38.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 msgr2=0x7f3dac198380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:28:38.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f3d980098e0 tx=0x7f3d9800f690 comp rx=0 tx=0).stop 2023-07-14T11:28:38.474 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 shutdown_connections 2023-07-14T11:28:38.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d9c0a12c0 0x7f3d9c0a3770 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3dac106fb0 0x7f3dac19c7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3dac101f70 0x7f3dac197e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 --2- 172.21.15.40:0/1002814319 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3dac102910 0x7f3dac198380 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:28:38.475 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 >> 172.21.15.40:0/1002814319 conn(0x7f3dac0fd7e0 msgr2=0x7f3dac0fe2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:28:38.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 shutdown_connections 2023-07-14T11:28:38.476 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:28:38.454+0000 7f3db2fff700 1 -- 172.21.15.40:0/1002814319 wait complete. 2023-07-14T11:28:38.477 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:28:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:39 smithi130 ceph-mon[129405]: pgmap v518: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:40.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:39 smithi040 ceph-mon[97954]: pgmap v518: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:40.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:39 smithi045 ceph-mon[128791]: pgmap v518: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:42.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:41 smithi130 ceph-mon[129405]: pgmap v519: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:42.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:41 smithi040 ceph-mon[97954]: pgmap v519: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:42.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:41 smithi045 ceph-mon[128791]: pgmap v519: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:44.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:43 smithi130 ceph-mon[129405]: pgmap v520: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:43 smithi040 ceph-mon[97954]: pgmap v520: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:44.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:43 smithi045 ceph-mon[128791]: pgmap v520: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:45 smithi130 ceph-mon[129405]: pgmap v521: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:46.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:45 smithi040 ceph-mon[97954]: pgmap v521: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:45 smithi045 ceph-mon[128791]: pgmap v521: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:48.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:47 smithi130 ceph-mon[129405]: pgmap v522: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:48.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:47 smithi040 ceph-mon[97954]: pgmap v522: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:48.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:47 smithi045 ceph-mon[128791]: pgmap v522: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:50.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:49 smithi130 ceph-mon[129405]: pgmap v523: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:49 smithi040 ceph-mon[97954]: pgmap v523: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:50.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:49 smithi045 ceph-mon[128791]: pgmap v523: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:52.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:51 smithi130 ceph-mon[129405]: pgmap v524: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:52.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:51 smithi040 ceph-mon[97954]: pgmap v524: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:52.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:51 smithi045 ceph-mon[128791]: pgmap v524: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:54.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:53 smithi130 ceph-mon[129405]: pgmap v525: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:54.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:53 smithi040 ceph-mon[97954]: pgmap v525: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:54.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:53 smithi045 ceph-mon[128791]: pgmap v525: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:56.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:55 smithi130 ceph-mon[129405]: pgmap v526: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:56.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:55 smithi040 ceph-mon[97954]: pgmap v526: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:56.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:55 smithi045 ceph-mon[128791]: pgmap v526: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:28:58.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:57 smithi130 ceph-mon[129405]: pgmap v527: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:58.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:57 smithi040 ceph-mon[97954]: pgmap v527: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:28:58.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:57 smithi045 ceph-mon[128791]: pgmap v527: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:28:59 smithi130 ceph-mon[129405]: pgmap v528: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:00.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:28:59 smithi040 ceph-mon[97954]: pgmap v528: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:00.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:28:59 smithi045 ceph-mon[128791]: pgmap v528: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:02.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:01 smithi130 ceph-mon[129405]: pgmap v529: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:02.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:01 smithi040 ceph-mon[97954]: pgmap v529: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:01 smithi045 ceph-mon[128791]: pgmap v529: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:03.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:02 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:29:03.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:02 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:29:03.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:02 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:29:04.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:03 smithi130 ceph-mon[129405]: pgmap v530: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:04.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:29:04.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:29:04.241 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:29:04.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:03 smithi040 ceph-mon[97954]: pgmap v530: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:04.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:29:04.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:29:04.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:29:04.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:03 smithi045 ceph-mon[128791]: pgmap v530: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:04.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:29:04.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:29:04.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:29:06.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:05 smithi130 ceph-mon[129405]: pgmap v531: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:29:06.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:05 smithi040 ceph-mon[97954]: pgmap v531: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:29:06.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:05 smithi045 ceph-mon[128791]: pgmap v531: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 44 KiB/s wr, 27 op/s 2023-07-14T11:29:08.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:07 smithi130 ceph-mon[129405]: pgmap v532: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:08.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:07 smithi040 ceph-mon[97954]: pgmap v532: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:08.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:07 smithi045 ceph-mon[128791]: pgmap v532: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:08.478 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status 2023-07-14T11:29:08.698 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 -- 172.21.15.40:0/465612155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 msgr2=0x7f8c3c105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 --2- 172.21.15.40:0/465612155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c105eb0 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 crypto rx=0x7f8c30009990 tx=0x7f8c30009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 -- 172.21.15.40:0/465612155 shutdown_connections 2023-07-14T11:29:08.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 --2- 172.21.15.40:0/465612155 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c100480 0x7f8c3c10c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 --2- 172.21.15.40:0/465612155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c105eb0 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.699 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 --2- 172.21.15.40:0/465612155 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c0fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 -- 172.21.15.40:0/465612155 >> 172.21.15.40:0/465612155 conn(0x7f8c3c0fb5f0 msgr2=0x7f8c3c0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 -- 172.21.15.40:0/465612155 shutdown_connections 2023-07-14T11:29:08.700 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.695+0000 7f8c42fff700 1 -- 172.21.15.40:0/465612155 wait complete. 2023-07-14T11:29:08.701 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.696+0000 7f8c42fff700 1 Processor -- start 2023-07-14T11:29:08.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 -- start start 2023-07-14T11:29:08.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c100480 0x7f8c3c198300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c19c760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.702 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3c19cda0 con 0x7f8c3c105ae0 2023-07-14T11:29:08.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3c19cf10 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c42fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c3c19d0d0 con 0x7f8c3c100480 2023-07-14T11:29:08.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36592/0 (socket says 172.21.15.40:36592) 2023-07-14T11:29:08.703 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3904606834 learned_addr learned my addr 172.21.15.40:0/3904606834 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:08.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c100480 0x7f8c3c198300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.697+0000 7f8c4159c700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c19c760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3904606834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c100480 msgr2=0x7f8c3c198300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c100480 0x7f8c3c198300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.704 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 msgr2=0x7f8c3c19c760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c19c760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3904606834 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c300095f0 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f8c30000c00 tx=0x7f8c30005740 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c39ffb700 1 -- 172.21.15.40:0/3904606834 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c3001d070 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.705 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c3c19d350 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c39ffb700 1 -- 172.21.15.40:0/3904606834 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8c30022470 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c39ffb700 1 -- 172.21.15.40:0/3904606834 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c30017650 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.706 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.698+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c3c1a4bf0 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.701+0000 7f8c39ffb700 1 -- 172.21.15.40:0/3904606834 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f8c300225e0 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.703+0000 7f8c569cbb80 1 Processor -- start 2023-07-14T11:29:08.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.703+0000 7f8c569cbb80 1 -- start start 2023-07-14T11:29:08.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 0x55c452ee56a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452ef0590 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef2eb0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452debcb0 con 0x55c452ef0ad0 2023-07-14T11:29:08.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452e3ea40 con 0x55c452ee52d0 2023-07-14T11:29:08.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452e37ed0 con 0x55c452ee5be0 2023-07-14T11:29:08.709 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452ef0590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c4159c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 0x55c452ee56a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c39ffb700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 0x7f8c2c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c4159c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 0x55c452ee56a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36594/0 (socket says 172.21.15.40:36594) 2023-07-14T11:29:08.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c4159c700 1 -- 172.21.15.40:0/3881582333 learned_addr learned my addr 172.21.15.40:0/3881582333 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:08.710 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef2eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.704+0000 7f8c39ffb700 1 -- 172.21.15.40:0/3904606834 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f8c300ce4f0 con 0x7f8c3c0ffad0 2023-07-14T11:29:08.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c4159c700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 0x7f8c2c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3881582333 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 msgr2=0x55c452ee56a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 0x55c452ee56a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3881582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 msgr2=0x55c452ef2eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef2eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 -- 172.21.15.40:0/3881582333 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c452c65de0 con 0x55c452ee5be0 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452ef0590 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f8c300cb850 tx=0x7f8c30022c80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=7b207c7437155db8 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c26ffd700 1 -- 172.21.15.40:0/3881582333 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c3009b030 con 0x55c452ee5be0 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c26ffd700 1 -- 172.21.15.40:0/3881582333 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8c3009ada0 con 0x55c452ee5be0 2023-07-14T11:29:08.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.705+0000 7f8c26ffd700 1 -- 172.21.15.40:0/3881582333 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c30099720 con 0x55c452ee5be0 2023-07-14T11:29:08.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3881582333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 msgr2=0x55c452ef0590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c4159c700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 0x7f8c2c0a3770 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f8c3400fb80 tx=0x7f8c34004b40 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452ef0590 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f8c300cb850 tx=0x7f8c30022c80 comp rx=0 tx=0).stop 2023-07-14T11:29:08.716 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3881582333 shutdown_connections 2023-07-14T11:29:08.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef2eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee52d0 0x55c452ee56a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3881582333 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452ef0590 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3881582333 >> 172.21.15.40:0/3881582333 conn(0x55c452edbda0 msgr2=0x55c452ee16a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.717 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3881582333 shutdown_connections 2023-07-14T11:29:08.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.706+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3881582333 wait complete. 2023-07-14T11:29:08.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.707+0000 7f8c569cbb80 1 -- start start 2023-07-14T11:29:08.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.707+0000 7f8c569cbb80 1 Processor -- start 2023-07-14T11:29:08.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.718 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee5be0 0x55c452ef6120 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452eefaa0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452dd7ac0 con 0x55c452ee5be0 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452dd77c0 con 0x55c452ef0ad0 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452cbb920 con 0x55c452ee52d0 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c569cbb80 10 client.0 _ll_register_callbacks cb 0x55c452dd5760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:29:08.719 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49344/0 (socket says 172.21.15.40:49344) 2023-07-14T11:29:08.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 -- 172.21.15.40:0/136959000 learned_addr learned my addr 172.21.15.40:0/136959000 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:08.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c4159c700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452eefaa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.720 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee5be0 0x55c452ef6120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 -- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 msgr2=0x55c452eefaa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.721 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452eefaa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 -- 172.21.15.40:0/136959000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee5be0 msgr2=0x55c452ef6120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.807 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee5be0 0x55c452ef6120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.708+0000 7f8c40d9b700 1 -- 172.21.15.40:0/136959000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c30095e70 con 0x55c452ee52d0 2023-07-14T11:29:08.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8c300ce7a0 tx=0x7f8c30097bd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=e51e4624b8a8b134 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 10 client.0 ms_handle_connect on v2:172.21.15.130:3300/0 2023-07-14T11:29:08.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c3009b030 con 0x55c452ee52d0 2023-07-14T11:29:08.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mon.1 v2:172.21.15.130:3300/0 2 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f8c3009f290 con 0x55c452ee52d0 2023-07-14T11:29:08.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x55c452d914f0 con 0x55c452ee52d0 2023-07-14T11:29:08.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8c3009abc0 con 0x55c452ee52d0 2023-07-14T11:29:08.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c569cbb80 10 client.26698 fetch_fsmap learned FSMap version 252 2023-07-14T11:29:08.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x55c452c4ec60 con 0x55c452ee52d0 2023-07-14T11:29:08.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mon.1 v2:172.21.15.130:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x55c452d914f0 con 0x55c452ee52d0 2023-07-14T11:29:08.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.709+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mon.1 v2:172.21.15.130:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8c3009b5e0 con 0x55c452ee52d0 2023-07-14T11:29:08.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 10 client.26698 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:29:08.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 10 client.26698 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:29:08.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 20 client.26698 populate_metadata read hostname 'smithi040' 2023-07-14T11:29:08.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452effba0 0x55c452f01f80 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 4 client.26698 mds_command: new command op to 24569 tid=0[{"prefix": "get_command_descriptions"}] 2023-07-14T11:29:08.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x55c452e30eb0 con 0x55c452effba0 2023-07-14T11:29:08.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452effba0 0x55c452f01f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452effba0 0x55c452f01f80 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=3895bb1e7a096fa8 server_cookie=b9ccbb01f8e2e05a in_seq=0 out_seq=0 2023-07-14T11:29:08.811 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.710+0000 7f8c1bfff700 10 client.26698 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:29:08.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c1bfff700 1 -- 172.21.15.40:0/136959000 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+18291 (crc 0 0 0) 0x55c452e30eb0 con 0x55c452effba0 2023-07-14T11:29:08.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c1bfff700 10 client.26698 handle_command_reply: tid=0 2023-07-14T11:29:08.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 client.26698 shutdown 2023-07-14T11:29:08.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 msgr2=0x55c452ef5be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.812 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f8c300ce7a0 tx=0x7f8c30097bd0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 shutdown_connections 2023-07-14T11:29:08.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452effba0 0x55c452f01f80 crc :-1 s=READY pgs=137 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452eefaa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.813 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452ef5be0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/136959000 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee5be0 0x55c452ef6120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.711+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 >> 172.21.15.40:0/136959000 conn(0x55c452edbda0 msgr2=0x55c452ee1290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.712+0000 7f8c1bfff700 0 client.26698 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:29:08.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.712+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 shutdown_connections 2023-07-14T11:29:08.814 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.712+0000 7f8c569cbb80 1 -- 172.21.15.40:0/136959000 wait complete. 2023-07-14T11:29:08.815 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.712+0000 7f8c569cbb80 20 client.26698 trim_cache size 0 max 16384 2023-07-14T11:29:08.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.729+0000 7f8c569cbb80 1 Processor -- start 2023-07-14T11:29:08.818 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.729+0000 7f8c569cbb80 1 -- start start 2023-07-14T11:29:08.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.729+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452eeed30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.729+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef57b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452ee6e30 con 0x55c452ef0ad0 2023-07-14T11:29:08.819 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452ee6320 con 0x55c452ee5be0 2023-07-14T11:29:08.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452ef0930 con 0x55c452ee52d0 2023-07-14T11:29:08.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452eeed30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36604/0 (socket says 172.21.15.40:36604) 2023-07-14T11:29:08.820 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 -- 172.21.15.40:0/3738899840 learned_addr learned my addr 172.21.15.40:0/3738899840 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:08.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c4159c700 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef57b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 -- 172.21.15.40:0/3738899840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 msgr2=0x55c452eeed30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452eeed30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 -- 172.21.15.40:0/3738899840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 msgr2=0x55c452ef57b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.821 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef57b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c3bfff700 1 -- 172.21.15.40:0/3738899840 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x55c452c8c250 con 0x55c452ee5be0 2023-07-14T11:29:08.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.730+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452eeed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:08.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8c28000c00 tx=0x7f8c28009260 comp rx=0 tx=0).ready entity=mon.2 client_cookie=5773cc448901799a server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c277fe700 1 -- 172.21.15.40:0/3738899840 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c28017070 con 0x55c452ee5be0 2023-07-14T11:29:08.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c277fe700 1 -- 172.21.15.40:0/3738899840 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8c28018040 con 0x55c452ee5be0 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c277fe700 1 -- 172.21.15.40:0/3738899840 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c280109f0 con 0x55c452ee5be0 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3738899840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 msgr2=0x55c452eefa50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 crypto rx=0x7f8c28000c00 tx=0x7f8c28009260 comp rx=0 tx=0).stop 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.731+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3738899840 shutdown_connections 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ef0ad0 0x55c452ef57b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee52d0 0x55c452eeed30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/3738899840 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ee5be0 0x55c452eefa50 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3738899840 >> 172.21.15.40:0/3738899840 conn(0x55c452edbda0 msgr2=0x55c452eef340 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3738899840 shutdown_connections 2023-07-14T11:29:08.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 -- 172.21.15.40:0/3738899840 wait complete. 2023-07-14T11:29:08.824 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 -- start start 2023-07-14T11:29:08.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.732+0000 7f8c569cbb80 1 Processor -- start 2023-07-14T11:29:08.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452f480a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452f4bd20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.825 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452f4c290 con 0x55c452ee52d0 2023-07-14T11:29:08.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452f4c400 con 0x55c452ef0ad0 2023-07-14T11:29:08.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x55c452f4c5c0 con 0x55c452ee5be0 2023-07-14T11:29:08.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c569cbb80 10 client.0 _ll_register_callbacks cb 0x55c452dd5760 invalidate_ino_cb 0 invalidate_dentry_cb 0 switch_interrupt_cb 0 remount_cb 0 2023-07-14T11:29:08.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.826 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35334/0 (socket says 172.21.15.40:35334) 2023-07-14T11:29:08.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 -- 172.21.15.40:0/2080741852 learned_addr learned my addr 172.21.15.40:0/2080741852 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:08.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452f480a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c4159c700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452f4bd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 -- 172.21.15.40:0/2080741852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 msgr2=0x55c452f480a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452f480a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 -- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 msgr2=0x55c452f4bd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=0).mark_down 2023-07-14T11:29:08.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452f4bd20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.733+0000 7f8c40d9b700 1 -- 172.21.15.40:0/2080741852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x55c452c4ec60 con 0x55c452ee52d0 2023-07-14T11:29:08.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c40d9b700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f8c3009af60 tx=0x7f8c30009ca0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=974fb631074823ea server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:08.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 10 client.0 ms_handle_connect on v2:172.21.15.40:3300/0 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8c300a3620 con 0x55c452ee52d0 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mon.0 v2:172.21.15.40:3300/0 2 ==== osd_map(113..113 src has 1..113) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f8c3009f290 con 0x55c452ee52d0 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_get_version(what=fsmap handle=1) v1 -- 0x7f8c3009ada0 con 0x55c452ee52d0 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_get_version_reply(handle=1 version=252) v2 ==== 24+0+0 (secure 0 0 0) 0x7f8c3007b070 con 0x55c452ee52d0 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 10 client.16680 fetch_fsmap learned FSMap version 252 2023-07-14T11:29:08.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,fsmap=252,monmap=4+}) v3 -- 0x7f8c28005ce0 con 0x55c452ee52d0 2023-07-14T11:29:08.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mon.0 v2:172.21.15.40:3300/0 4 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8c3009ada0 con 0x55c452ee52d0 2023-07-14T11:29:08.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mon.0 v2:172.21.15.40:3300/0 5 ==== fsmap(e 252) v1 ==== 7470+0+0 (secure 0 0 0) 0x7f8c3009b550 con 0x55c452ee52d0 2023-07-14T11:29:08.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 10 client.16680 fetch_fsmap finished waiting for FSMap version 252 2023-07-14T11:29:08.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 10 client.16680 resolve_mds: resolved 1:0 to role '1:0' aka daemon mds.b 2023-07-14T11:29:08.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 20 client.16680 populate_metadata read hostname 'smithi040' 2023-07-14T11:29:08.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.734+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452f54b50 0x55c452f56f60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:08.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.735+0000 7f8c569cbb80 4 client.16680 mds_command: new command op to 24569 tid=0[{"prefix": "scrub status"}] 2023-07-14T11:29:08.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.735+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 --> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] -- command(tid 0: {"prefix": "scrub status"}) v1 -- 0x55c452df3050 con 0x55c452f54b50 2023-07-14T11:29:08.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.735+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452f54b50 0x55c452f56f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:08.831 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.735+0000 7f8c3bfff700 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452f54b50 0x55c452f56f60 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=mds.0 client_cookie=c138a3a4d61fd1bc server_cookie=89e0b9f05a1e29fe in_seq=0 out_seq=0 2023-07-14T11:29:08.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.735+0000 7f8c1bfff700 10 client.16680 ms_handle_connect on v2:172.21.15.45:6838/180621035 2023-07-14T11:29:08.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c1bfff700 1 -- 172.21.15.40:0/2080741852 <== mds.0 v2:172.21.15.45:6838/180621035 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+265 (crc 0 0 0) 0x55c452df3050 con 0x55c452f54b50 2023-07-14T11:29:08.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c1bfff700 10 client.16680 handle_command_reply: tid=0 2023-07-14T11:29:08.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 client.16680 shutdown 2023-07-14T11:29:08.832 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 msgr2=0x55c452f47b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 crypto rx=0x7f8c3009af60 tx=0x7f8c30009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 shutdown_connections 2023-07-14T11:29:08.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:6838/180621035,v1:172.21.15.45:6839/180621035] conn(0x55c452f54b50 0x55c452f56f60 crc :-1 s=READY pgs=138 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.833 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x55c452ef0ad0 0x55c452f4bd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x55c452ee52d0 0x55c452f47b60 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c1bfff700 0 client.16680 ms_handle_reset on v2:172.21.15.45:6838/180621035 2023-07-14T11:29:08.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 --2- 172.21.15.40:0/2080741852 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x55c452ee5be0 0x55c452f480a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 >> 172.21.15.40:0/2080741852 conn(0x55c452edbda0 msgr2=0x55c452ee7c70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.834 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 shutdown_connections 2023-07-14T11:29:08.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 1 -- 172.21.15.40:0/2080741852 wait complete. 2023-07-14T11:29:08.835 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.736+0000 7f8c569cbb80 20 client.16680 trim_cache size 0 max 16384 2023-07-14T11:29:08.835 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:08.836 INFO:teuthology.orchestra.run.smithi040.stdout: "status": "scrub active (2 inodes in the stack)", 2023-07-14T11:29:08.836 INFO:teuthology.orchestra.run.smithi040.stdout: "scrubs": { 2023-07-14T11:29:08.836 INFO:teuthology.orchestra.run.smithi040.stdout: "4caa05a5-53a5-4cfd-80c5-659be1c8bc93": { 2023-07-14T11:29:08.836 INFO:teuthology.orchestra.run.smithi040.stdout: "path": "/", 2023-07-14T11:29:08.836 INFO:teuthology.orchestra.run.smithi040.stdout: "tag": "4caa05a5-53a5-4cfd-80c5-659be1c8bc93", 2023-07-14T11:29:08.837 INFO:teuthology.orchestra.run.smithi040.stdout: "options": "recursive,force" 2023-07-14T11:29:08.837 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:08.837 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:08.837 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:08.838 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 msgr2=0x7f8c2c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 0x7f8c2c0a3770 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f8c3400fb80 tx=0x7f8c34004b40 comp rx=0 tx=0).stop 2023-07-14T11:29:08.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 msgr2=0x7f8c3c197dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:08.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 crypto rx=0x7f8c30000c00 tx=0x7f8c30005740 comp rx=0 tx=0).stop 2023-07-14T11:29:08.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 shutdown_connections 2023-07-14T11:29:08.839 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8c2c0a12c0 0x7f8c2c0a3770 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c3c105ae0 0x7f8c3c19c760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8c3c0ffad0 0x7f8c3c197dc0 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 --2- 172.21.15.40:0/3904606834 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8c3c100480 0x7f8c3c198300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:08.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 >> 172.21.15.40:0/3904606834 conn(0x7f8c3c0fb5f0 msgr2=0x7f8c3c10ad20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:08.840 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 shutdown_connections 2023-07-14T11:29:08.841 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:08.737+0000 7f8c42fff700 1 -- 172.21.15.40:0/3904606834 wait complete. 2023-07-14T11:29:08.841 INFO:tasks.cephfs.filesystem:scrub status for tag:4caa05a5-53a5-4cfd-80c5-659be1c8bc93 - {'path': '/', 'tag': '4caa05a5-53a5-4cfd-80c5-659be1c8bc93', 'options': 'recursive,force'} 2023-07-14T11:29:08.841 ERROR:tasks.fwd_scrub.fs.[cephfs]:exception: Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/fwd_scrub.py", line 38, in _run self.do_scrub() File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/fwd_scrub.py", line 55, in do_scrub self._scrub() File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/fwd_scrub.py", line 76, in _scrub done = self.fs.wait_until_scrub_complete(tag=tag, sleep=30, timeout=self.scrub_timeout) File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/cephfs/filesystem.py", line 1684, in wait_until_scrub_complete while proceed(): File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/contextutil.py", line 134, in __call__ raise MaxWhileTries(error_msg) teuthology.exceptions.MaxWhileTries: reached maximum tries (31) after waiting for 900 seconds 2023-07-14T11:29:08.853 ERROR:teuthology.run_tasks:Manager failed: fwd_scrub Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_aa6a80e6cc655fff48452a7eca02070e61d074fc/teuthology/run_tasks.py", line 192, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/fwd_scrub.py", line 164, in task stop_all_fwd_scrubbers(ctx.ceph[config['cluster']].thrashers) File "/home/teuthworker/src/git.ceph.com_ceph-c_3fc0089cc1082313a98853d48b892acdc1385b8b/qa/tasks/fwd_scrub.py", line 99, in stop_all_fwd_scrubbers raise RuntimeError(f"error during scrub thrashing: {thrasher.exception}") RuntimeError: error during scrub thrashing: reached maximum tries (31) after waiting for 900 seconds 2023-07-14T11:29:08.857 DEBUG:teuthology.run_tasks:Unwinding manager check-counter 2023-07-14T11:29:08.868 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:29:09.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 -- 172.21.15.40:0/3064987976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c0731c0 msgr2=0x7fed2c10bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.127 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 --2- 172.21.15.40:0/3064987976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c0731c0 0x7fed2c10bdd0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7fed1c009990 tx=0x7fed1c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 -- 172.21.15.40:0/3064987976 shutdown_connections 2023-07-14T11:29:09.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 --2- 172.21.15.40:0/3064987976 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c10c310 0x7fed2c10e6f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 --2- 172.21.15.40:0/3064987976 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0728b0 0x7fed2c072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 --2- 172.21.15.40:0/3064987976 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c0731c0 0x7fed2c10bdd0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.128 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 -- 172.21.15.40:0/3064987976 >> 172.21.15.40:0/3064987976 conn(0x7fed2c0fb590 msgr2=0x7fed2c0fd9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:09.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 -- 172.21.15.40:0/3064987976 shutdown_connections 2023-07-14T11:29:09.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.124+0000 7fed33bff700 1 -- 172.21.15.40:0/3064987976 wait complete. 2023-07-14T11:29:09.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.126+0000 7fed33bff700 1 Processor -- start 2023-07-14T11:29:09.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.126+0000 7fed33bff700 1 -- start start 2023-07-14T11:29:09.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 0x7fed2c197da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 0x7fed2c19c740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed2c19cd80 con 0x7fed2c0728b0 2023-07-14T11:29:09.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed2c19cef0 con 0x7fed2c0731c0 2023-07-14T11:29:09.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed33bff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed2c19d0b0 con 0x7fed2c10c310 2023-07-14T11:29:09.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 0x7fed2c197da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3119a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3119a700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36634/0 (socket says 172.21.15.40:36634) 2023-07-14T11:29:09.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3219c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 0x7fed2c19c740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3119a700 1 -- 172.21.15.40:0/1852348104 learned_addr learned my addr 172.21.15.40:0/1852348104 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:09.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 -- 172.21.15.40:0/1852348104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 msgr2=0x7fed2c19c740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 0x7fed2c19c740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 -- 172.21.15.40:0/1852348104 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 msgr2=0x7fed2c1982e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3199b700 1 -- 172.21.15.40:0/1852348104 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed1c0095f0 con 0x7fed2c0728b0 2023-07-14T11:29:09.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3119a700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:09.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.127+0000 7fed3219c700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 0x7fed2c19c740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:09.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed3199b700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 0x7fed2c197da0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7fed2800e9f0 tx=0x7fed2800ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:09.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed2800c9c0 con 0x7fed2c0728b0 2023-07-14T11:29:09.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fed2c19d390 con 0x7fed2c0728b0 2023-07-14T11:29:09.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fed28010430 con 0x7fed2c0728b0 2023-07-14T11:29:09.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fed28018680 con 0x7fed2c0728b0 2023-07-14T11:29:09.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.128+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fed2c1a4bd0 con 0x7fed2c0728b0 2023-07-14T11:29:09.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.129+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fed2c106c30 con 0x7fed2c0728b0 2023-07-14T11:29:09.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.131+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fed28010860 con 0x7fed2c0728b0 2023-07-14T11:29:09.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.132+0000 7fed22ffd700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 0x7fed180a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.132+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7fed28014070 con 0x7fed2c0728b0 2023-07-14T11:29:09.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.133+0000 7fed3119a700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 0x7fed180a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.133+0000 7fed3119a700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 0x7fed180a3830 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fed2c1993b0 tx=0x7fed1c00b260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:09.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.136+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fed28051790 con 0x7fed2c0728b0 2023-07-14T11:29:09.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.348+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fed2c067300 con 0x7fed2c0728b0 2023-07-14T11:29:09.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.350+0000 7fed22ffd700 1 -- 172.21.15.40:0/1852348104 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v114) v1 ==== 74+0+25182 (secure 0 0 0) 0x7fed280961e0 con 0x7fed2c0728b0 2023-07-14T11:29:09.353 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:29:09.354 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":114,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:29:08.866807+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:0/583014137":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/234853232":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/3580297519":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3852248366":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]},{"pool":4,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]}],"new_removed_snaps":[{"pool":3,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]},{"pool":4,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:29:09.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.352+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 msgr2=0x7fed180a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.352+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 0x7fed180a3830 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fed2c1993b0 tx=0x7fed1c00b260 comp rx=0 tx=0).stop 2023-07-14T11:29:09.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.352+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 msgr2=0x7fed2c197da0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.352+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 0x7fed2c197da0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 crypto rx=0x7fed2800e9f0 tx=0x7fed2800ed00 comp rx=0 tx=0).stop 2023-07-14T11:29:09.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 shutdown_connections 2023-07-14T11:29:09.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fed180a1380 0x7fed180a3830 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fed2c10c310 0x7fed2c19c740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fed2c0728b0 0x7fed2c197da0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 --2- 172.21.15.40:0/1852348104 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fed2c0731c0 0x7fed2c1982e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 >> 172.21.15.40:0/1852348104 conn(0x7fed2c0fb590 msgr2=0x7fed2c103d70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:09.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 shutdown_connections 2023-07-14T11:29:09.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.353+0000 7fed33bff700 1 -- 172.21.15.40:0/1852348104 wait complete. 2023-07-14T11:29:09.374 INFO:tasks.check_counter.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '114', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 29, 'snap_epoch': 114, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 8197, 'flags_names': 'hashpspool,ec_overwrites,selfmanaged_snaps', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '114', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 29, 'snap_epoch': 114, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:29:09.376 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:29:09.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 -- 172.21.15.40:0/2894216933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 msgr2=0x7f3874105ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.584 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 --2- 172.21.15.40:0/2894216933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f3874105ec0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f386800b0a0 tx=0x7f386800b3b0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 -- 172.21.15.40:0/2894216933 shutdown_connections 2023-07-14T11:29:09.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 --2- 172.21.15.40:0/2894216933 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f387410c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 --2- 172.21.15.40:0/2894216933 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f3874105ec0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 --2- 172.21.15.40:0/2894216933 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f38740fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 -- 172.21.15.40:0/2894216933 >> 172.21.15.40:0/2894216933 conn(0x7f38740fb5e0 msgr2=0x7f38740fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:09.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 -- 172.21.15.40:0/2894216933 shutdown_connections 2023-07-14T11:29:09.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.581+0000 7f387b1ff700 1 -- 172.21.15.40:0/2894216933 wait complete. 2023-07-14T11:29:09.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.582+0000 7f387b1ff700 1 Processor -- start 2023-07-14T11:29:09.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.582+0000 7f387b1ff700 1 -- start start 2023-07-14T11:29:09.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f3874197f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f387419cf30 con 0x7f38740ffae0 2023-07-14T11:29:09.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f387419d0a0 con 0x7f3874100490 2023-07-14T11:29:09.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387b1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f387419d260 con 0x7f3874105af0 2023-07-14T11:29:09.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387979c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36654/0 (socket says 172.21.15.40:36654) 2023-07-14T11:29:09.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f387979c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49394/0 (socket says 172.21.15.40:49394) 2023-07-14T11:29:09.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3878f9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f3874197f50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 -- 172.21.15.40:0/1252651659 learned_addr learned my addr 172.21.15.40:0/1252651659 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:09.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 -- 172.21.15.40:0/1252651659 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 msgr2=0x7f387419c8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 -- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 msgr2=0x7f3874197f50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.583+0000 7f3873fff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f3874197f50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3873fff700 1 -- 172.21.15.40:0/1252651659 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3868009d00 con 0x7f3874100490 2023-07-14T11:29:09.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f387979c700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:09.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3878f9b700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f3874197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:09.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3873fff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f386000e9a0 tx=0x7f386000ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:09.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3860009790 con 0x7f3874100490 2023-07-14T11:29:09.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f387419d540 con 0x7f3874100490 2023-07-14T11:29:09.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3860010430 con 0x7f3874100490 2023-07-14T11:29:09.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3860018680 con 0x7f3874100490 2023-07-14T11:29:09.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.584+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f38741a4d80 con 0x7f3874100490 2023-07-14T11:29:09.594 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.585+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3874067290 con 0x7f3874100490 2023-07-14T11:29:09.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.588+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f38600105a0 con 0x7f3874100490 2023-07-14T11:29:09.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.589+0000 7f3871ffb700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 0x7f38640a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:09.595 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.589+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7f3860014070 con 0x7f3874100490 2023-07-14T11:29:09.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.589+0000 7f3878f9b700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 0x7f38640a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:09.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.590+0000 7f3878f9b700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 0x7f38640a3770 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f3868009fd0 tx=0x7f3868006210 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:09.596 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.592+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f38600964d0 con 0x7f3874100490 2023-07-14T11:29:09.804 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.801+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f38741a5010 con 0x7f3874100490 2023-07-14T11:29:09.805 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.802+0000 7f3871ffb700 1 -- 172.21.15.40:0/1252651659 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v114) v1 ==== 93+0+10 (secure 0 0 0) 0x7f38600260a0 con 0x7f3874100490 2023-07-14T11:29:09.805 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:29:09.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 msgr2=0x7f38640a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 0x7f38640a3770 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f3868009fd0 tx=0x7f3868006210 comp rx=0 tx=0).stop 2023-07-14T11:29:09.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 msgr2=0x7f3874198490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:09.808 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 crypto rx=0x7f386000e9a0 tx=0x7f386000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 shutdown_connections 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f38640a12c0 0x7f38640a3770 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3874105af0 0x7f387419c8f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f38740ffae0 0x7f3874197f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 --2- 172.21.15.40:0/1252651659 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3874100490 0x7f3874198490 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:09.809 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 >> 172.21.15.40:0/1252651659 conn(0x7f38740fb5e0 msgr2=0x7f387410b030 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:09.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 shutdown_connections 2023-07-14T11:29:09.810 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:09.805+0000 7f387b1ff700 1 -- 172.21.15.40:0/1252651659 wait complete. 2023-07-14T11:29:09.827 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:29:10.037 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 -- 172.21.15.40:0/4248595986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 msgr2=0x7fc87410c910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 --2- 172.21.15.40:0/4248595986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc87410c910 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fc86c00b0a0 tx=0x7fc86c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 -- 172.21.15.40:0/4248595986 shutdown_connections 2023-07-14T11:29:10.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 --2- 172.21.15.40:0/4248595986 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc87410c910 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 --2- 172.21.15.40:0/4248595986 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc874105ae0 0x7fc874105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.038 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 --2- 172.21.15.40:0/4248595986 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc8740ffad0 0x7fc8740fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 -- 172.21.15.40:0/4248595986 >> 172.21.15.40:0/4248595986 conn(0x7fc8740fb5f0 msgr2=0x7fc8740fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:10.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.034+0000 7fc879dff700 1 -- 172.21.15.40:0/4248595986 shutdown_connections 2023-07-14T11:29:10.039 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.035+0000 7fc879dff700 1 -- 172.21.15.40:0/4248595986 wait complete. 2023-07-14T11:29:10.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 Processor -- start 2023-07-14T11:29:10.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 -- start start 2023-07-14T11:29:10.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8740ffad0 0x7fc874197f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.041 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc874105ae0 0x7fc87419c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc87419cee0 con 0x7fc8740ffad0 2023-07-14T11:29:10.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc87419d050 con 0x7fc874105ae0 2023-07-14T11:29:10.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc879dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc87419d210 con 0x7fc874100480 2023-07-14T11:29:10.042 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc872ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc873fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc874105ae0 0x7fc87419c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc872ffd700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49420/0 (socket says 172.21.15.40:49420) 2023-07-14T11:29:10.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc872ffd700 1 -- 172.21.15.40:0/1741160453 learned_addr learned my addr 172.21.15.40:0/1741160453 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:10.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.036+0000 7fc8737fe700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8740ffad0 0x7fc874197f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.043 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 -- 172.21.15.40:0/1741160453 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc874105ae0 msgr2=0x7fc87419c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc874105ae0 0x7fc87419c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 -- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8740ffad0 msgr2=0x7fc874197f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8740ffad0 0x7fc874197f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 -- 172.21.15.40:0/1741160453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc86c009d00 con 0x7fc874100480 2023-07-14T11:29:10.044 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc872ffd700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fc86800e9a0 tx=0x7fc86800ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc868009790 con 0x7fc874100480 2023-07-14T11:29:10.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fc868010430 con 0x7fc874100480 2023-07-14T11:29:10.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fc868018680 con 0x7fc874100480 2023-07-14T11:29:10.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc87419d4f0 con 0x7fc874100480 2023-07-14T11:29:10.045 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.037+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc8741a4d30 con 0x7fc874100480 2023-07-14T11:29:10.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.038+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc874067290 con 0x7fc874100480 2023-07-14T11:29:10.046 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.041+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fc8680105a0 con 0x7fc874100480 2023-07-14T11:29:10.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.042+0000 7fc870ff9700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 0x7fc85c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.042+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7fc868014070 con 0x7fc874100480 2023-07-14T11:29:10.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.042+0000 7fc8737fe700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 0x7fc85c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.043+0000 7fc8737fe700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 0x7fc85c0a3770 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fc864009be0 tx=0x7fc864009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.047 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.044+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fc868096290 con 0x7fc874100480 2023-07-14T11:29:10.256 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.254+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7fc8741a4fc0 con 0x7fc874100480 2023-07-14T11:29:10.257 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.254+0000 7fc870ff9700 1 -- 172.21.15.40:0/1741160453 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v114) v1 ==== 104+0+11 (secure 0 0 0) 0x7fc8680998b0 con 0x7fc874100480 2023-07-14T11:29:10.257 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:29:10.259 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 msgr2=0x7fc85c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 0x7fc85c0a3770 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fc864009be0 tx=0x7fc864009260 comp rx=0 tx=0).stop 2023-07-14T11:29:10.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 msgr2=0x7fc874198440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fc86800e9a0 tx=0x7fc86800ecb0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.260 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 shutdown_connections 2023-07-14T11:29:10.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7fc85c0a12c0 0x7fc85c0a3770 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fc874105ae0 0x7fc87419c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8740ffad0 0x7fc874197f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 --2- 172.21.15.40:0/1741160453 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fc874100480 0x7fc874198440 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.261 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 >> 172.21.15.40:0/1741160453 conn(0x7fc8740fb5f0 msgr2=0x7fc87410ae70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:10.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 shutdown_connections 2023-07-14T11:29:10.262 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.257+0000 7fc879dff700 1 -- 172.21.15.40:0/1741160453 wait complete. 2023-07-14T11:29:10.276 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:09 smithi040 ceph-mon[97954]: pgmap v533: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:10.276 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:09 smithi040 ceph-mon[97954]: osdmap e114: 12 total, 12 up, 12 in 2023-07-14T11:29:10.276 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1852348104' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:10.276 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:09 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1252651659' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:10.278 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:29:10.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:09 smithi130 ceph-mon[129405]: pgmap v533: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:10.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:09 smithi130 ceph-mon[129405]: osdmap e114: 12 total, 12 up, 12 in 2023-07-14T11:29:10.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1852348104' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:10.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:09 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1252651659' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:09 smithi045 ceph-mon[128791]: pgmap v533: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 13 KiB/s rd, 44 KiB/s wr, 23 op/s 2023-07-14T11:29:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:09 smithi045 ceph-mon[128791]: osdmap e114: 12 total, 12 up, 12 in 2023-07-14T11:29:10.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1852348104' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:10.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:09 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1252651659' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:10.487 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 -- 172.21.15.40:0/4057206556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 msgr2=0x7f4280105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.488 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 --2- 172.21.15.40:0/4057206556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f4280105eb0 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 crypto rx=0x7f427c009990 tx=0x7f427c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 -- 172.21.15.40:0/4057206556 shutdown_connections 2023-07-14T11:29:10.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 --2- 172.21.15.40:0/4057206556 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f428010c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 --2- 172.21.15.40:0/4057206556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f4280105eb0 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 --2- 172.21.15.40:0/4057206556 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f42800fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.489 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 -- 172.21.15.40:0/4057206556 >> 172.21.15.40:0/4057206556 conn(0x7f42800fb5f0 msgr2=0x7f42800fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:10.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 -- 172.21.15.40:0/4057206556 shutdown_connections 2023-07-14T11:29:10.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.484+0000 7f42885ff700 1 -- 172.21.15.40:0/4057206556 wait complete. 2023-07-14T11:29:10.490 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.485+0000 7f42885ff700 1 Processor -- start 2023-07-14T11:29:10.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.485+0000 7f42885ff700 1 -- start start 2023-07-14T11:29:10.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.492 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f42801a3dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4280073580 con 0x7f4280105ae0 2023-07-14T11:29:10.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42801a4300 con 0x7f42800ffad0 2023-07-14T11:29:10.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f42885ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42801a4440 con 0x7f4280100480 2023-07-14T11:29:10.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f428639b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.493 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:49424/0 (socket says 172.21.15.40:49424) 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 -- 172.21.15.40:0/3133506944 learned_addr learned my addr 172.21.15.40:0/3133506944 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f428639b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36688/0 (socket says 172.21.15.40:36688) 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4286b9c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f42801a3dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 -- 172.21.15.40:0/3133506944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 msgr2=0x7f4280072810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.494 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 -- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 msgr2=0x7f42801a3dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f42801a3dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4285b9a700 1 -- 172.21.15.40:0/3133506944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f427c0095f0 con 0x7f4280100480 2023-07-14T11:29:10.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f428639b700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:10.495 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.486+0000 7f4286b9c700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f42801a3dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:10.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f4285b9a700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f427000e9a0 tx=0x7f427000ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4270009790 con 0x7f4280100480 2023-07-14T11:29:10.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42801a46a0 con 0x7f4280100480 2023-07-14T11:29:10.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4270010430 con 0x7f4280100480 2023-07-14T11:29:10.496 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4270018680 con 0x7f4280100480 2023-07-14T11:29:10.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.487+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42801a4b80 con 0x7f4280100480 2023-07-14T11:29:10.497 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.488+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4280067290 con 0x7f4280100480 2023-07-14T11:29:10.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.490+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f42700105a0 con 0x7f4280100480 2023-07-14T11:29:10.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.491+0000 7f42777fe700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 0x7f426c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.491+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7f4270014070 con 0x7f4280100480 2023-07-14T11:29:10.498 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.492+0000 7f428639b700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 0x7f426c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.492+0000 7f428639b700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 0x7f426c0a3770 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f427c000c00 tx=0x7f427c00b260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.499 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.493+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4270096290 con 0x7f4280100480 2023-07-14T11:29:10.707 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.705+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f42801a4ed0 con 0x7f4280100480 2023-07-14T11:29:10.708 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.705+0000 7f42777fe700 1 -- 172.21.15.40:0/3133506944 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v114) v1 ==== 100+0+11 (secure 0 0 0) 0x7f42700998b0 con 0x7f4280100480 2023-07-14T11:29:10.709 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:29:10.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 msgr2=0x7f426c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 0x7f426c0a3770 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f427c000c00 tx=0x7f427c00b260 comp rx=0 tx=0).stop 2023-07-14T11:29:10.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 msgr2=0x7f4280072d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7f427000e9a0 tx=0x7f427000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.711 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 shutdown_connections 2023-07-14T11:29:10.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f426c0a12c0 0x7f426c0a3770 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4280105ae0 0x7f42801a3dc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f42800ffad0 0x7f4280072810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 --2- 172.21.15.40:0/3133506944 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4280100480 0x7f4280072d50 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.712 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 >> 172.21.15.40:0/3133506944 conn(0x7f42800fb5f0 msgr2=0x7f42800fbbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:10.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 shutdown_connections 2023-07-14T11:29:10.713 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.708+0000 7f42885ff700 1 -- 172.21.15.40:0/3133506944 wait complete. 2023-07-14T11:29:10.729 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:29:10.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 -- 172.21.15.40:0/2773679158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c0ffb00 msgr2=0x7f4d6c0fff70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 --2- 172.21.15.40:0/2773679158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c0fff70 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7f4d600098f0 tx=0x7f4d60009c00 comp rx=0 tx=0).stop 2023-07-14T11:29:10.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 -- 172.21.15.40:0/2773679158 shutdown_connections 2023-07-14T11:29:10.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 --2- 172.21.15.40:0/2773679158 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c1004b0 0x7f4d6c10c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 --2- 172.21.15.40:0/2773679158 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c105b10 0x7f4d6c105ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 --2- 172.21.15.40:0/2773679158 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c0fff70 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 -- 172.21.15.40:0/2773679158 >> 172.21.15.40:0/2773679158 conn(0x7f4d6c0fb600 msgr2=0x7f4d6c0fda10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:10.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 -- 172.21.15.40:0/2773679158 shutdown_connections 2023-07-14T11:29:10.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.938+0000 7f4d727ff700 1 -- 172.21.15.40:0/2773679158 wait complete. 2023-07-14T11:29:10.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.939+0000 7f4d727ff700 1 Processor -- start 2023-07-14T11:29:10.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 -- start start 2023-07-14T11:29:10.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c1004b0 0x7f4d6c19bfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 0x7f4d6c195ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d6c196140 con 0x7f4d6c105b10 2023-07-14T11:29:10.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d6c1962b0 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d727ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d6c196470 con 0x7f4d6c1004b0 2023-07-14T11:29:10.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d6bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d6bfff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36706/0 (socket says 172.21.15.40:36706) 2023-07-14T11:29:10.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d6bfff700 1 -- 172.21.15.40:0/1099852310 learned_addr learned my addr 172.21.15.40:0/1099852310 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:10.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.940+0000 7f4d6b7fe700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c1004b0 0x7f4d6c19bfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d70d9c700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 0x7f4d6c195ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 -- 172.21.15.40:0/1099852310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c1004b0 msgr2=0x7f4d6c19bfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c1004b0 0x7f4d6c19bfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 -- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 msgr2=0x7f4d6c195ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 0x7f4d6c195ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 -- 172.21.15.40:0/1099852310 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d64009c40 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d70d9c700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 0x7f4d6c195ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2023-07-14T11:29:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d6bfff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f4d5c00ec10 tx=0x7f4d5c00ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.950 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c00cd40 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d5c0095f0 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4d5c0044e0 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4d5c010690 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.941+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d60009520 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.951 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.942+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d6c067290 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.952 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.945+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f4d5c003710 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.946+0000 7f4d697fa700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 0x7f4d540a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.946+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7f4d5c014070 con 0x7f4d6c0ffb00 2023-07-14T11:29:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.946+0000 7f4d6b7fe700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 0x7f4d540a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.947+0000 7f4d6b7fe700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 0x7f4d540a3770 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f4d60005e90 tx=0x7f4d60019040 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:10.953 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:10.949+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4d5c096790 con 0x7f4d6c0ffb00 2023-07-14T11:29:11.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:10 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1741160453' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:10 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3133506944' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.158+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f4d6c196fe0 con 0x7f4d6c0ffb00 2023-07-14T11:29:11.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.159+0000 7f4d697fa700 1 -- 172.21.15.40:0/1099852310 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v114) v1 ==== 103+0+11 (secure 0 0 0) 0x7f4d5c099db0 con 0x7f4d6c0ffb00 2023-07-14T11:29:11.163 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:29:11.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 msgr2=0x7f4d540a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:11.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 0x7f4d540a3770 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f4d60005e90 tx=0x7f4d60019040 comp rx=0 tx=0).stop 2023-07-14T11:29:11.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 msgr2=0x7f4d6c19ba60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:11.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 crypto rx=0x7f4d5c00ec10 tx=0x7f4d5c00ef20 comp rx=0 tx=0).stop 2023-07-14T11:29:11.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 shutdown_connections 2023-07-14T11:29:11.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4d540a12c0 0x7f4d540a3770 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d6c105b10 0x7f4d6c195ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4d6c0ffb00 0x7f4d6c19ba60 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 --2- 172.21.15.40:0/1099852310 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4d6c1004b0 0x7f4d6c19bfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 >> 172.21.15.40:0/1099852310 conn(0x7f4d6c0fb600 msgr2=0x7f4d6c0fbc20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:11.168 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.161+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 shutdown_connections 2023-07-14T11:29:11.169 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.162+0000 7f4d727ff700 1 -- 172.21.15.40:0/1099852310 wait complete. 2023-07-14T11:29:11.183 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json-pretty 2023-07-14T11:29:11.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:10 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1741160453' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:10 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3133506944' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:10 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1741160453' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:10 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3133506944' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:29:11.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 -- 172.21.15.40:0/3943031157 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 msgr2=0x7f8904105ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:11.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 --2- 172.21.15.40:0/3943031157 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f8904105ed0 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7f88f8009990 tx=0x7f88f8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 -- 172.21.15.40:0/3943031157 shutdown_connections 2023-07-14T11:29:11.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 --2- 172.21.15.40:0/3943031157 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 0x7f890410c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 --2- 172.21.15.40:0/3943031157 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f8904105ed0 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 --2- 172.21.15.40:0/3943031157 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f89040fff60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 -- 172.21.15.40:0/3943031157 >> 172.21.15.40:0/3943031157 conn(0x7f89040fb630 msgr2=0x7f89040fda40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:11.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 -- 172.21.15.40:0/3943031157 shutdown_connections 2023-07-14T11:29:11.390 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.385+0000 7f890abff700 1 -- 172.21.15.40:0/3943031157 wait complete. 2023-07-14T11:29:11.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 Processor -- start 2023-07-14T11:29:11.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 -- start start 2023-07-14T11:29:11.392 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:11.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 0x7f89041982d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:11.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f890419c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:11.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.386+0000 7f890abff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890419cd70 con 0x7f89041004a0 2023-07-14T11:29:11.393 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890abff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890419cee0 con 0x7f89040ffaf0 2023-07-14T11:29:11.394 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890abff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f890419d0a0 con 0x7f8904105b00 2023-07-14T11:29:11.394 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:11.394 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:36732/0 (socket says 172.21.15.40:36732) 2023-07-14T11:29:11.394 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 -- 172.21.15.40:0/3217748928 learned_addr learned my addr 172.21.15.40:0/3217748928 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:11.394 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f8903fff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 0x7f89041982d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:11.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890919c700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f890419c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:11.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 -- 172.21.15.40:0/3217748928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 msgr2=0x7f890419c730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:11.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f890419c730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 -- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 msgr2=0x7f89041982d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:11.395 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 0x7f89041982d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:11.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 -- 172.21.15.40:0/3217748928 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88f80095f0 con 0x7f89040ffaf0 2023-07-14T11:29:11.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.387+0000 7f890899b700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f88f8005720 tx=0x7f88f8005750 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:11.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.388+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f88f801d070 con 0x7f89040ffaf0 2023-07-14T11:29:11.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.388+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f88f8022470 con 0x7f89040ffaf0 2023-07-14T11:29:11.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.388+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f890419d320 con 0x7f89040ffaf0 2023-07-14T11:29:11.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.388+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f88f8017650 con 0x7f89040ffaf0 2023-07-14T11:29:11.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.388+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89041a4bc0 con 0x7f89040ffaf0 2023-07-14T11:29:11.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.389+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8904067300 con 0x7f89040ffaf0 2023-07-14T11:29:11.398 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.391+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f88f80225e0 con 0x7f89040ffaf0 2023-07-14T11:29:11.398 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.393+0000 7f8901ffb700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 0x7f88f40a37b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:11.398 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.393+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(114..114 src has 1..114) v4 ==== 9520+0+0 (secure 0 0 0) 0x7f88f80ce460 con 0x7f89040ffaf0 2023-07-14T11:29:11.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.393+0000 7f8903fff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 0x7f88f40a37b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:11.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.394+0000 7f8903fff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 0x7f88f40a37b0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f89041993a0 tx=0x7f88f0009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:11.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.398+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f88f80974e0 con 0x7f89040ffaf0 2023-07-14T11:29:11.669 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.667+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json-pretty"} v 0) v1 -- 0x7f89041a4d50 con 0x7f89040ffaf0 2023-07-14T11:29:11.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.678+0000 7f8901ffb700 1 -- 172.21.15.40:0/3217748928 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json-pretty"}]=0 v24) v1 ==== 81+0+733775 (secure 0 0 0) 0x7f88f809ab00 con 0x7f89040ffaf0 2023-07-14T11:29:11.690 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:29:11.691 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:11.691 INFO:teuthology.orchestra.run.smithi040.stdout: "epoch": 24, 2023-07-14T11:29:11.691 INFO:teuthology.orchestra.run.smithi040.stdout: "active_gid": 15582, 2023-07-14T11:29:11.691 INFO:teuthology.orchestra.run.smithi040.stdout: "active_name": "x", 2023-07-14T11:29:11.692 INFO:teuthology.orchestra.run.smithi040.stdout: "active_addrs": { 2023-07-14T11:29:11.692 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:11.692 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.692 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:11.693 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:6800", 2023-07-14T11:29:11.693 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 4243839420 2023-07-14T11:29:11.693 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.693 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.693 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v1", 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:6801", 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 4243839420 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.694 INFO:teuthology.orchestra.run.smithi040.stdout: "active_addr": "172.21.15.40:6801/4243839420", 2023-07-14T11:29:11.695 INFO:teuthology.orchestra.run.smithi040.stdout: "active_change": "2023-07-14T11:12:08.502172+0000", 2023-07-14T11:29:11.695 INFO:teuthology.orchestra.run.smithi040.stdout: "active_mgr_features": 4540138322906710015, 2023-07-14T11:29:11.695 INFO:teuthology.orchestra.run.smithi040.stdout: "available": true, 2023-07-14T11:29:11.695 INFO:teuthology.orchestra.run.smithi040.stdout: "standbys": [ 2023-07-14T11:29:11.695 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.696 INFO:teuthology.orchestra.run.smithi040.stdout: "gid": 25570, 2023-07-14T11:29:11.696 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "z", 2023-07-14T11:29:11.696 INFO:teuthology.orchestra.run.smithi040.stdout: "mgr_features": 4540138322906710015, 2023-07-14T11:29:11.696 INFO:teuthology.orchestra.run.smithi040.stdout: "available_modules": [ 2023-07-14T11:29:11.696 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.697 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alerts", 2023-07-14T11:29:11.697 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:11.697 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:11.697 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:11.697 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:11.698 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:11.698 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.698 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.698 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.698 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:11.699 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.699 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.699 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.699 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How frequently to reexamine health status", 2023-07-14T11:29:11.699 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.700 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.700 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.700 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.700 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:11.700 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.701 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.702 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.702 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.702 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.702 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.702 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.703 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.703 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.703 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.703 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.703 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.704 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.704 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.704 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.704 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:11.704 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:11.705 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.705 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.705 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.705 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.705 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.706 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.706 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.706 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.706 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.706 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.707 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.708 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.708 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:11.708 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.708 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.708 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.709 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.709 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.709 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.709 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.709 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.710 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.710 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.710 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.710 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.710 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.711 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.711 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.711 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:11.711 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:11.711 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.712 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.712 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.712 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.712 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.712 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.713 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.713 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.713 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.713 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.713 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.714 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.714 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_destination": { 2023-07-14T11:29:11.714 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_destination", 2023-07-14T11:29:11.714 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.714 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.715 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email address to send alerts to", 2023-07-14T11:29:11.716 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.716 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.716 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.716 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.716 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_from_name": { 2023-07-14T11:29:11.717 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_from_name", 2023-07-14T11:29:11.717 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.717 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.717 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.717 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "Ceph", 2023-07-14T11:29:11.718 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.718 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.718 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.718 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email From: name", 2023-07-14T11:29:11.718 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.719 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.719 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.719 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.719 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_host": { 2023-07-14T11:29:11.719 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_host", 2023-07-14T11:29:11.720 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.720 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.720 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.720 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.720 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.721 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.721 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.721 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP server", 2023-07-14T11:29:11.721 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.721 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.722 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.722 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.722 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_password": { 2023-07-14T11:29:11.722 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_password", 2023-07-14T11:29:11.722 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.723 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.723 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.723 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.723 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.723 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.724 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.724 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Password to authenticate with", 2023-07-14T11:29:11.724 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.724 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.724 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.725 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.725 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_port": { 2023-07-14T11:29:11.725 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_port", 2023-07-14T11:29:11.725 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.725 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "465", 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.726 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP port", 2023-07-14T11:29:11.727 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.727 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.727 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.727 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.727 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_sender": { 2023-07-14T11:29:11.728 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_sender", 2023-07-14T11:29:11.728 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.728 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.728 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.728 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.729 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.729 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.729 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.729 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP envelope sender", 2023-07-14T11:29:11.729 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.730 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.730 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.730 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.730 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_ssl": { 2023-07-14T11:29:11.731 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_ssl", 2023-07-14T11:29:11.731 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.731 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.731 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.731 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.732 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.732 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.732 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.732 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use SSL to connect to SMTP server", 2023-07-14T11:29:11.733 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.733 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.733 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.733 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.734 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_user": { 2023-07-14T11:29:11.734 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_user", 2023-07-14T11:29:11.734 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.734 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.734 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.735 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.735 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.735 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.735 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.735 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "User to authenticate as", 2023-07-14T11:29:11.736 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.736 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.736 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.736 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.736 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.737 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.737 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.737 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "balancer", 2023-07-14T11:29:11.737 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:11.737 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:11.738 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:11.738 INFO:teuthology.orchestra.run.smithi040.stdout: "active": { 2023-07-14T11:29:11.738 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "active", 2023-07-14T11:29:11.738 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.738 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.739 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.739 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.739 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.739 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.739 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.740 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically balance PGs across cluster", 2023-07-14T11:29:11.740 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.740 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.740 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.740 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.741 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_time": { 2023-07-14T11:29:11.741 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_time", 2023-07-14T11:29:11.741 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.741 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.741 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.742 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0000", 2023-07-14T11:29:11.742 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.742 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.742 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.743 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "beginning time of day to automatically balance", 2023-07-14T11:29:11.743 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:11.743 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.743 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.743 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.744 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_weekday": { 2023-07-14T11:29:11.744 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_weekday", 2023-07-14T11:29:11.744 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:11.744 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.745 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.745 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:11.745 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:11.745 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:11.745 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.746 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2023-07-14T11:29:11.746 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:11.746 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.746 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.746 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.747 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_max_iterations": { 2023-07-14T11:29:11.747 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_max_iterations", 2023-07-14T11:29:11.747 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:11.747 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.747 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.748 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "25", 2023-07-14T11:29:11.748 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:11.748 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "250", 2023-07-14T11:29:11.748 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.748 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum number of iterations to attempt optimization", 2023-07-14T11:29:11.749 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.749 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.749 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.749 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.749 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_metrics": { 2023-07-14T11:29:11.750 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_metrics", 2023-07-14T11:29:11.750 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.750 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.750 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.750 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "pgs,objects,bytes", 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "metrics with which to calculate OSD utilization", 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2023-07-14T11:29:11.751 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.752 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.752 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.752 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_step": { 2023-07-14T11:29:11.752 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_step", 2023-07-14T11:29:11.752 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:11.753 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.753 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.753 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.5", 2023-07-14T11:29:11.753 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0.001", 2023-07-14T11:29:11.753 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "0.999", 2023-07-14T11:29:11.754 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.754 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "aggressiveness of optimization", 2023-07-14T11:29:11.754 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2023-07-14T11:29:11.754 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.754 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.755 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.755 INFO:teuthology.orchestra.run.smithi040.stdout: "end_time": { 2023-07-14T11:29:11.755 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_time", 2023-07-14T11:29:11.755 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.755 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2359", 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.756 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ending time of day to automatically balance", 2023-07-14T11:29:11.757 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:11.757 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.757 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.757 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "end_weekday": { 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_weekday", 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.758 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:11.759 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:11.759 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:11.759 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.759 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2023-07-14T11:29:11.759 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:11.760 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.760 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.760 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.760 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:11.760 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:11.761 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.761 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.761 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.761 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.761 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.762 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.762 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.762 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.762 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.762 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.763 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.763 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.763 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.763 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.763 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.764 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.764 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.764 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.764 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.764 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:11.765 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:11.765 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.765 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.765 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.765 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.766 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.766 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.766 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.766 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.766 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.767 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.767 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.767 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.767 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:11.767 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:11.768 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.768 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.768 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.768 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:11.768 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.769 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.769 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.769 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.769 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.769 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.770 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.770 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.770 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.770 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.771 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:11.772 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:11.772 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.772 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.772 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.773 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.773 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.773 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.773 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.773 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.774 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.774 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.774 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.774 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.774 INFO:teuthology.orchestra.run.smithi040.stdout: "min_score": { 2023-07-14T11:29:11.775 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_score", 2023-07-14T11:29:11.775 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:11.775 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.775 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.775 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "minimum score, below which no optimization is attempted", 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.776 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.777 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.777 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.777 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:11.777 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:11.777 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "upmap", 2023-07-14T11:29:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: "crush-compat", 2023-07-14T11:29:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: "none", 2023-07-14T11:29:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap" 2023-07-14T11:29:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Balancer mode", 2023-07-14T11:29:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.781 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_ids": { 2023-07-14T11:29:11.781 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_ids", 2023-07-14T11:29:11.781 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.781 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.781 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.782 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.782 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.782 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.782 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.782 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "pools which the automatic balancing will be limited to", 2023-07-14T11:29:11.783 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.783 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.783 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.783 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.783 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:11.784 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:11.784 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.784 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.784 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.784 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:11.785 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.785 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.785 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.785 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and attempt optimization", 2023-07-14T11:29:11.785 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.786 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.786 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.786 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.786 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_deviation": { 2023-07-14T11:29:11.786 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_deviation", 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:11.787 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.788 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.788 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "deviation below which no optimization is attempted", 2023-07-14T11:29:11.788 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2023-07-14T11:29:11.788 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.789 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.789 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.789 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_optimizations": { 2023-07-14T11:29:11.789 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_optimizations", 2023-07-14T11:29:11.789 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:11.790 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.790 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.790 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:11.790 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.791 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.791 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.791 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum upmap optimizations to make per attempt", 2023-07-14T11:29:11.791 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.791 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.792 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.792 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.792 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.792 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.792 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.793 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cephadm", 2023-07-14T11:29:11.793 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:11.793 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:11.793 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:11.793 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_down_multiplier": { 2023-07-14T11:29:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_down_multiplier", 2023-07-14T11:29:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:11.795 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.795 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.795 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.795 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2023-07-14T11:29:11.795 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.796 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.796 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.796 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.796 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_refresh_rate": { 2023-07-14T11:29:11.796 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_refresh_rate", 2023-07-14T11:29:11.797 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.797 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.797 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.797 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "20", 2023-07-14T11:29:11.797 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.798 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.798 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.798 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2023-07-14T11:29:11.798 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.798 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.799 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.799 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.799 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_starting_port": { 2023-07-14T11:29:11.799 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_starting_port", 2023-07-14T11:29:11.799 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.800 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.800 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.800 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4721", 2023-07-14T11:29:11.800 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.800 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.801 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.801 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2023-07-14T11:29:11.801 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.801 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.801 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.802 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.802 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_password": { 2023-07-14T11:29:11.802 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_password", 2023-07-14T11:29:11.802 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.803 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.803 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.803 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:11.803 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.803 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.804 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.804 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web password", 2023-07-14T11:29:11.804 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.804 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.804 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.805 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.805 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_user": { 2023-07-14T11:29:11.805 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_user", 2023-07-14T11:29:11.805 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.805 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.806 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.806 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:11.806 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.806 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.807 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.807 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web user", 2023-07-14T11:29:11.807 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.807 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.807 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.808 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.808 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_ptrace": { 2023-07-14T11:29:11.808 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_ptrace", 2023-07-14T11:29:11.809 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.809 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.809 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.809 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.809 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.810 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.810 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.810 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2023-07-14T11:29:11.810 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2023-07-14T11:29:11.810 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_interval": { 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_interval", 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.811 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.812 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.812 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:11.812 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.812 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.813 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.813 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to autotune daemon memory", 2023-07-14T11:29:11.813 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.813 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.813 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.814 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.814 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_memory_target_ratio": { 2023-07-14T11:29:11.814 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_memory_target_ratio", 2023-07-14T11:29:11.814 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:11.814 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.815 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.815 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.7", 2023-07-14T11:29:11.815 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.815 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.815 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.816 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2023-07-14T11:29:11.816 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.816 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.816 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.816 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.817 INFO:teuthology.orchestra.run.smithi040.stdout: "cgroups_split": { 2023-07-14T11:29:11.817 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cgroups_split", 2023-07-14T11:29:11.817 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.817 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.817 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.818 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.818 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.818 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.818 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.818 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2023-07-14T11:29:11.819 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.819 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.819 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.819 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.819 INFO:teuthology.orchestra.run.smithi040.stdout: "config_checks_enabled": { 2023-07-14T11:29:11.820 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_checks_enabled", 2023-07-14T11:29:11.820 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.820 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.820 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.820 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.821 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.821 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.821 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.821 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2023-07-14T11:29:11.821 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.822 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.822 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.822 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.822 INFO:teuthology.orchestra.run.smithi040.stdout: "config_dashboard": { 2023-07-14T11:29:11.822 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_dashboard", 2023-07-14T11:29:11.823 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.823 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.823 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.823 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.823 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.824 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.824 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.824 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2023-07-14T11:29:11.824 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.824 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.825 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.825 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.825 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_alertmanager": { 2023-07-14T11:29:11.825 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_alertmanager", 2023-07-14T11:29:11.825 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.826 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.826 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.826 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2023-07-14T11:29:11.826 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.826 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.827 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.828 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_base": { 2023-07-14T11:29:11.828 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_base", 2023-07-14T11:29:11.828 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.828 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.828 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.829 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph", 2023-07-14T11:29:11.829 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.829 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.829 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Container image name, without the tag", 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.830 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_elasticsearch": { 2023-07-14T11:29:11.831 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_elasticsearch", 2023-07-14T11:29:11.831 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.831 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.831 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.831 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2023-07-14T11:29:11.832 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.832 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.832 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.832 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "elasticsearch container image", 2023-07-14T11:29:11.832 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.833 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.833 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.833 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.833 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_grafana": { 2023-07-14T11:29:11.834 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_grafana", 2023-07-14T11:29:11.834 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.834 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.834 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.834 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2023-07-14T11:29:11.835 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.835 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.835 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.835 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:11.835 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.836 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.836 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.836 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.836 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_haproxy": { 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_haproxy", 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2023-07-14T11:29:11.837 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.838 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.838 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.838 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "HAproxy container image", 2023-07-14T11:29:11.838 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.838 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.839 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.839 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.839 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_agent": { 2023-07-14T11:29:11.839 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_agent", 2023-07-14T11:29:11.839 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.840 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.840 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.840 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2023-07-14T11:29:11.840 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.840 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.841 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.841 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger agent container image", 2023-07-14T11:29:11.841 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.841 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.841 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.842 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.842 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_collector": { 2023-07-14T11:29:11.842 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_collector", 2023-07-14T11:29:11.842 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.842 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.843 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.843 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2023-07-14T11:29:11.843 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.843 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.843 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.844 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger collector container image", 2023-07-14T11:29:11.844 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.844 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.844 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.844 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.845 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_query": { 2023-07-14T11:29:11.845 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_query", 2023-07-14T11:29:11.845 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.845 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.845 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.846 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2023-07-14T11:29:11.846 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.846 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.846 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.846 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger query container image", 2023-07-14T11:29:11.847 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.847 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.847 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.847 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.847 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_keepalived": { 2023-07-14T11:29:11.848 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_keepalived", 2023-07-14T11:29:11.848 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.848 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.848 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.848 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2023-07-14T11:29:11.849 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.849 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.849 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.849 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Keepalived container image", 2023-07-14T11:29:11.849 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.850 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.850 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.850 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.850 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_loki": { 2023-07-14T11:29:11.850 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_loki", 2023-07-14T11:29:11.851 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.851 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.851 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.851 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2023-07-14T11:29:11.851 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.852 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.852 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.852 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Loki container image", 2023-07-14T11:29:11.852 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.852 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_node_exporter": { 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_node_exporter", 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.853 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.854 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.854 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2023-07-14T11:29:11.854 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.854 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.855 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.855 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:11.855 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.855 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.855 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.856 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.856 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_prometheus": { 2023-07-14T11:29:11.856 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_prometheus", 2023-07-14T11:29:11.856 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.856 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.857 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:11.858 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.858 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.858 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.858 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.858 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_promtail": { 2023-07-14T11:29:11.859 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_promtail", 2023-07-14T11:29:11.859 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.859 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.859 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.859 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2023-07-14T11:29:11.860 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.860 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.860 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.860 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Promtail container image", 2023-07-14T11:29:11.860 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.861 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.861 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.861 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.861 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_snmp_gateway": { 2023-07-14T11:29:11.861 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_snmp_gateway", 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.862 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.863 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.863 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SNMP Gateway container image", 2023-07-14T11:29:11.863 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.863 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.863 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.864 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.864 INFO:teuthology.orchestra.run.smithi040.stdout: "container_init": { 2023-07-14T11:29:11.864 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_init", 2023-07-14T11:29:11.864 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.864 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.865 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.865 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.865 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.865 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.865 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.866 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Run podman/docker with `--init`", 2023-07-14T11:29:11.866 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.866 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.866 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.866 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.867 INFO:teuthology.orchestra.run.smithi040.stdout: "daemon_cache_timeout": { 2023-07-14T11:29:11.867 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "daemon_cache_timeout", 2023-07-14T11:29:11.867 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.867 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.867 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.868 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:11.868 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.868 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.868 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.868 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache service (daemon) inventory", 2023-07-14T11:29:11.869 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.869 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.869 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.869 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.870 INFO:teuthology.orchestra.run.smithi040.stdout: "default_cephadm_command_timeout": { 2023-07-14T11:29:11.870 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_cephadm_command_timeout", 2023-07-14T11:29:11.870 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.870 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.870 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.871 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "900", 2023-07-14T11:29:11.871 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.871 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.871 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.871 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2023-07-14T11:29:11.872 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.872 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.872 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.872 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.872 INFO:teuthology.orchestra.run.smithi040.stdout: "default_registry": { 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_registry", 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io", 2023-07-14T11:29:11.873 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.874 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.874 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.874 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2023-07-14T11:29:11.874 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.874 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: "device_cache_timeout": { 2023-07-14T11:29:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_cache_timeout", 2023-07-14T11:29:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1800", 2023-07-14T11:29:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.877 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.877 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.877 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache device inventory", 2023-07-14T11:29:11.877 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.877 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.878 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.878 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.878 INFO:teuthology.orchestra.run.smithi040.stdout: "device_enhanced_scan": { 2023-07-14T11:29:11.878 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_enhanced_scan", 2023-07-14T11:29:11.879 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.879 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.879 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.879 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.879 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.880 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.880 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.880 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use libstoragemgmt during device scans", 2023-07-14T11:29:11.881 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.881 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.881 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.881 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.881 INFO:teuthology.orchestra.run.smithi040.stdout: "facts_cache_timeout": { 2023-07-14T11:29:11.882 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "facts_cache_timeout", 2023-07-14T11:29:11.882 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.882 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.882 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.882 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:11.883 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.883 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.883 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.883 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache host facts data", 2023-07-14T11:29:11.883 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.884 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.884 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.884 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.884 INFO:teuthology.orchestra.run.smithi040.stdout: "host_check_interval": { 2023-07-14T11:29:11.884 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "host_check_interval", 2023-07-14T11:29:11.885 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.885 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.885 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.885 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:11.885 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.886 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.886 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.886 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to perform a host check", 2023-07-14T11:29:11.886 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:11.887 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.888 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.888 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.888 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.888 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.888 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.889 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.889 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.889 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.889 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.890 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.891 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.891 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.891 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.891 INFO:teuthology.orchestra.run.smithi040.stdout: "log_refresh_metadata": { 2023-07-14T11:29:11.891 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_refresh_metadata", 2023-07-14T11:29:11.892 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.892 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.892 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.892 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.892 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.893 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.893 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.893 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2023-07-14T11:29:11.893 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.893 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.894 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.894 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.894 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:11.894 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:11.894 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.895 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.895 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.895 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.895 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.895 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.896 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.897 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:11.897 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:11.897 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.897 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.898 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.899 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.899 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.899 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.899 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.899 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.900 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.900 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.900 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.900 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.900 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.901 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.901 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:11.901 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:11.901 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.901 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.902 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.902 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.902 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.902 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.902 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.903 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.903 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.903 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.903 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.903 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf": { 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf", 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.904 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.905 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.905 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.905 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.905 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2023-07-14T11:29:11.906 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.906 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.906 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.906 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.906 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "*", 2023-07-14T11:29:11.907 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.908 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.908 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.908 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2023-07-14T11:29:11.908 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.908 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.909 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.909 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.909 INFO:teuthology.orchestra.run.smithi040.stdout: "max_count_per_host": { 2023-07-14T11:29:11.909 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_count_per_host", 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.910 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.911 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.911 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of daemons per service per host", 2023-07-14T11:29:11.911 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.911 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.911 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.912 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.912 INFO:teuthology.orchestra.run.smithi040.stdout: "max_osd_draining_count": { 2023-07-14T11:29:11.912 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_osd_draining_count", 2023-07-14T11:29:11.912 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.912 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.913 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.913 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:11.913 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.913 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.913 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.914 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2023-07-14T11:29:11.914 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.914 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.914 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.914 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "migration_current": { 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "migration_current", 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.915 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.916 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.916 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.916 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.916 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "internal - do not modify", 2023-07-14T11:29:11.916 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.917 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.917 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.917 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.917 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:11.917 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:11.918 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.918 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.918 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.918 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "root", 2023-07-14T11:29:11.918 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.919 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.919 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.919 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm-package", 2023-07-14T11:29:11.919 INFO:teuthology.orchestra.run.smithi040.stdout: "root" 2023-07-14T11:29:11.919 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.920 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "mode for remote execution of cephadm", 2023-07-14T11:29:11.920 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.920 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.920 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.920 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.921 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_alerts_path": { 2023-07-14T11:29:11.921 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_alerts_path", 2023-07-14T11:29:11.921 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.921 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.921 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.922 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2023-07-14T11:29:11.922 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.922 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.922 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.922 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "location of alerts to include in prometheus deployments", 2023-07-14T11:29:11.923 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.923 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.923 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.923 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.923 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_password": { 2023-07-14T11:29:11.924 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_password", 2023-07-14T11:29:11.924 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.924 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.924 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.924 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web password", 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.925 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.926 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.926 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.926 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_user": { 2023-07-14T11:29:11.926 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_user", 2023-07-14T11:29:11.926 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.927 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.927 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.927 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:11.927 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.927 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.928 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.928 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web user", 2023-07-14T11:29:11.928 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.928 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.928 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.929 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.929 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_insecure": { 2023-07-14T11:29:11.929 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_insecure", 2023-07-14T11:29:11.929 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.929 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.930 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.930 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.930 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.930 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.930 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.931 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2023-07-14T11:29:11.931 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.931 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.931 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.931 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_password": { 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_password", 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.932 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.933 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.933 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.933 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.933 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2023-07-14T11:29:11.933 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.934 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.934 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.934 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.934 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_url": { 2023-07-14T11:29:11.934 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_url", 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.935 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.936 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.936 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2023-07-14T11:29:11.936 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.936 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.936 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.937 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.937 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_username": { 2023-07-14T11:29:11.937 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_username", 2023-07-14T11:29:11.937 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.937 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.938 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.938 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.938 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.938 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.938 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.939 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2023-07-14T11:29:11.939 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.939 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.939 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.940 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.940 INFO:teuthology.orchestra.run.smithi040.stdout: "secure_monitoring_stack": { 2023-07-14T11:29:11.940 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "secure_monitoring_stack", 2023-07-14T11:29:11.940 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.940 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.941 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2023-07-14T11:29:11.942 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.942 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.942 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.942 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.942 INFO:teuthology.orchestra.run.smithi040.stdout: "service_discovery_port": { 2023-07-14T11:29:11.943 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "service_discovery_port", 2023-07-14T11:29:11.943 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.943 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.943 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.943 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8765", 2023-07-14T11:29:11.944 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.944 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.944 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.944 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "cephadm service discovery port", 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: "ssh_config_file": { 2023-07-14T11:29:11.945 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssh_config_file", 2023-07-14T11:29:11.946 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.946 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.946 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.946 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.947 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.947 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.947 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.947 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "customized SSH config file to connect to managed hosts", 2023-07-14T11:29:11.947 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: "use_agent": { 2023-07-14T11:29:11.948 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_agent", 2023-07-14T11:29:11.949 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.949 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.949 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.949 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.949 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.950 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.950 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.950 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2023-07-14T11:29:11.950 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.950 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.951 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.951 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.951 INFO:teuthology.orchestra.run.smithi040.stdout: "use_repo_digest": { 2023-07-14T11:29:11.951 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_repo_digest", 2023-07-14T11:29:11.951 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.952 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.952 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.952 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.952 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.952 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.953 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.953 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2023-07-14T11:29:11.953 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.953 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.953 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.954 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.954 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_failed_host_check": { 2023-07-14T11:29:11.954 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_failed_host_check", 2023-07-14T11:29:11.954 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.954 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.955 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.955 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.955 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.955 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.955 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.956 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if the host check fails", 2023-07-14T11:29:11.956 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.956 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.956 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.956 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.957 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_daemons": { 2023-07-14T11:29:11.957 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_daemons", 2023-07-14T11:29:11.957 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.957 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.957 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2023-07-14T11:29:11.958 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.959 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.959 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.959 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.959 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_hosts": { 2023-07-14T11:29:11.960 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_hosts", 2023-07-14T11:29:11.960 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.960 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.960 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.960 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.961 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.961 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.961 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.961 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2023-07-14T11:29:11.961 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.962 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.963 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crash", 2023-07-14T11:29:11.963 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:11.963 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:11.963 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:11.963 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:11.964 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:11.964 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.964 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.964 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.964 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.965 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.965 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.965 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.965 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.965 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.966 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.966 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.966 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.966 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.966 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.967 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:11.968 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:11.968 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.968 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.968 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.968 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.969 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.969 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.969 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.969 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.969 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.970 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.970 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.970 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.970 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:11.970 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:11.971 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.971 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.971 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.971 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:11.971 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.972 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.972 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:11.972 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:11.972 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:11.972 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:11.973 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:11.973 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:11.973 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:11.973 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:11.973 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.974 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.974 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.974 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.974 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.974 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:11.975 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:11.975 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.975 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.975 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.975 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.976 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.976 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.976 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.976 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: "retain_interval": { 2023-07-14T11:29:11.977 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retain_interval", 2023-07-14T11:29:11.978 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.978 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.978 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.978 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "31536000", 2023-07-14T11:29:11.978 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.979 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.979 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.979 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain crashes before pruning them", 2023-07-14T11:29:11.979 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.979 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.980 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.980 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.980 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_recent_interval": { 2023-07-14T11:29:11.980 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_recent_interval", 2023-07-14T11:29:11.980 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:11.981 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.981 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:11.981 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1209600", 2023-07-14T11:29:11.981 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.981 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.982 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.982 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "time interval in which to warn about recent crashes", 2023-07-14T11:29:11.982 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.982 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.982 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dashboard", 2023-07-14T11:29:11.983 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:11.984 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:11.984 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:11.984 INFO:teuthology.orchestra.run.smithi040.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2023-07-14T11:29:11.984 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2023-07-14T11:29:11.984 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:11.985 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.985 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.985 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:11.985 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.985 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.986 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.986 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.986 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.987 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.987 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.987 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.987 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_HOST": { 2023-07-14T11:29:11.987 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_HOST", 2023-07-14T11:29:11.988 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:11.988 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.988 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.988 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:11.988 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.989 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.989 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.989 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.989 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.989 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.990 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.991 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.991 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.991 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.991 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.991 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.992 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.992 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.992 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.992 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.992 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.993 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_ENABLED": { 2023-07-14T11:29:11.993 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_ENABLED", 2023-07-14T11:29:11.993 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.993 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.993 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.994 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:11.994 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.994 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.994 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.994 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_LOG_PAYLOAD": { 2023-07-14T11:29:11.995 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2023-07-14T11:29:11.996 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.996 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.996 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.996 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.996 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.997 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:11.997 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:11.997 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:11.997 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:11.997 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:11.998 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:11.998 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:11.998 INFO:teuthology.orchestra.run.smithi040.stdout: "ENABLE_BROWSABLE_API": { 2023-07-14T11:29:11.998 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ENABLE_BROWSABLE_API", 2023-07-14T11:29:11.998 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:11.999 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:11.999 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:11.999 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:11.999 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:11.999 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.000 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.000 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.000 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.000 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.000 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.001 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.001 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_CEPHFS": { 2023-07-14T11:29:12.001 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2023-07-14T11:29:12.001 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.001 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.002 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.002 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.002 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.002 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.002 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.003 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2023-07-14T11:29:12.004 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2023-07-14T11:29:12.004 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.004 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.004 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.005 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.005 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.005 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.005 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.005 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_ISCSI": { 2023-07-14T11:29:12.006 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2023-07-14T11:29:12.007 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.007 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.007 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.007 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.007 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.008 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.008 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.008 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.008 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.008 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.009 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.009 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.009 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_MIRRORING": { 2023-07-14T11:29:12.009 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2023-07-14T11:29:12.009 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.010 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.010 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.010 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.010 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.010 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.011 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.011 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.011 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.011 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.012 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.012 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.012 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_NFS": { 2023-07-14T11:29:12.012 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_NFS", 2023-07-14T11:29:12.012 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.013 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.014 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.014 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.014 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.014 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.014 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.015 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RBD": { 2023-07-14T11:29:12.015 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RBD", 2023-07-14T11:29:12.015 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.015 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.015 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.016 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.016 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.016 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.016 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.016 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.017 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.017 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.017 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.017 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.017 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RGW": { 2023-07-14T11:29:12.018 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RGW", 2023-07-14T11:29:12.018 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.018 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.018 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.018 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.019 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.020 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.020 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.020 INFO:teuthology.orchestra.run.smithi040.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2023-07-14T11:29:12.020 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2023-07-14T11:29:12.021 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.021 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.021 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.021 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.021 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.022 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.022 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.022 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.022 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.022 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_PASSWORD": { 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_PASSWORD", 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.023 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.024 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.024 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:12.024 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.024 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.024 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.025 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.025 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.025 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.025 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.025 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.026 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_SSL_VERIFY": { 2023-07-14T11:29:12.026 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2023-07-14T11:29:12.026 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.026 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.026 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.027 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.027 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.027 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.027 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.027 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.028 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.028 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.028 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.028 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.028 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_URL": { 2023-07-14T11:29:12.029 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_URL", 2023-07-14T11:29:12.029 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.029 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.029 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.029 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.030 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.030 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.030 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.030 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.030 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.031 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.031 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.031 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.031 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_USERNAME": { 2023-07-14T11:29:12.031 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_USERNAME", 2023-07-14T11:29:12.032 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.032 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.032 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.032 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:12.032 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.033 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.033 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.033 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.033 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.033 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.034 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.034 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.034 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_FRONTEND_API_URL": { 2023-07-14T11:29:12.034 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2023-07-14T11:29:12.034 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.035 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.035 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.035 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.035 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.035 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.036 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.036 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.036 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.036 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.036 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.037 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.038 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.038 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.038 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.038 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.038 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.039 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.039 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.039 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.039 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.039 INFO:teuthology.orchestra.run.smithi040.stdout: "ISCSI_API_SSL_VERIFICATION": { 2023-07-14T11:29:12.040 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2023-07-14T11:29:12.040 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.040 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.040 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.040 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.041 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.041 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.041 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.041 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.041 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: "ISSUE_TRACKER_API_KEY": { 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISSUE_TRACKER_API_KEY", 2023-07-14T11:29:12.042 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.043 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.043 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.043 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.043 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.043 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.044 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.044 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.044 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.044 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.044 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.045 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.045 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_HOST": { 2023-07-14T11:29:12.045 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_HOST", 2023-07-14T11:29:12.045 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.045 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.046 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.046 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.046 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.046 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.046 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.047 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.047 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.047 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.047 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.047 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.048 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2023-07-14T11:29:12.048 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2023-07-14T11:29:12.048 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.048 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.049 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.049 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.049 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.049 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.049 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.050 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.050 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.050 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.050 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.050 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.051 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2023-07-14T11:29:12.051 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2023-07-14T11:29:12.051 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.051 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.051 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.052 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.052 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.052 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.052 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.052 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2023-07-14T11:29:12.053 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2023-07-14T11:29:12.054 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.054 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.054 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.054 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.054 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.055 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.055 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.055 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.055 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.055 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.056 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.056 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.056 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2023-07-14T11:29:12.056 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2023-07-14T11:29:12.056 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.057 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.057 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.057 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.057 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.057 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.058 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.058 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.058 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.058 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.058 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.059 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.059 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2023-07-14T11:29:12.059 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2023-07-14T11:29:12.059 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.059 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.060 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.061 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.061 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.061 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.061 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.061 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2023-07-14T11:29:12.062 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2023-07-14T11:29:12.062 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.062 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.062 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.062 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.063 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.063 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.063 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.063 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.063 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.064 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.064 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.064 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.064 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2023-07-14T11:29:12.064 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2023-07-14T11:29:12.065 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.065 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.065 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.065 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.065 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.066 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.066 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.066 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.066 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.066 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.067 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.067 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.067 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2023-07-14T11:29:12.067 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2023-07-14T11:29:12.067 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.068 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.068 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.068 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.068 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.068 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.069 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.069 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.069 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.069 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.069 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.070 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.070 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_ENABLED": { 2023-07-14T11:29:12.070 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_ENABLED", 2023-07-14T11:29:12.070 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.070 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.071 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.071 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.071 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.071 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.071 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.072 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.072 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.072 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.072 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.072 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.073 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2023-07-14T11:29:12.073 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2023-07-14T11:29:12.073 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.073 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.073 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.074 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2023-07-14T11:29:12.074 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.074 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.074 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.075 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.075 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.075 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.075 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.075 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.076 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2023-07-14T11:29:12.076 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2023-07-14T11:29:12.076 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.076 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.076 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.077 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.078 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.078 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.078 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.078 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_LENGTH": { 2023-07-14T11:29:12.078 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2023-07-14T11:29:12.079 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.079 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.079 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.079 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8", 2023-07-14T11:29:12.079 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.080 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.080 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.080 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.080 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.080 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.081 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.081 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.081 INFO:teuthology.orchestra.run.smithi040.stdout: "REST_REQUESTS_TIMEOUT": { 2023-07-14T11:29:12.081 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "REST_REQUESTS_TIMEOUT", 2023-07-14T11:29:12.081 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "45", 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.082 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.083 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.083 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.083 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.083 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ACCESS_KEY": { 2023-07-14T11:29:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ACCESS_KEY", 2023-07-14T11:29:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.084 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.085 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.086 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.086 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.086 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.086 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.086 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.087 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ADMIN_RESOURCE": { 2023-07-14T11:29:12.087 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2023-07-14T11:29:12.087 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.087 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.087 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.088 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:12.088 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.088 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.088 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.088 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.089 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.089 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.089 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.089 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.089 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SECRET_KEY": { 2023-07-14T11:29:12.090 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SECRET_KEY", 2023-07-14T11:29:12.090 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.090 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.090 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.090 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.091 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.092 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.092 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.092 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SSL_VERIFY": { 2023-07-14T11:29:12.092 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SSL_VERIFY", 2023-07-14T11:29:12.092 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.093 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.093 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.093 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.093 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.093 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.094 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.094 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.094 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.094 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.094 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.095 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.095 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_SPAN": { 2023-07-14T11:29:12.095 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2023-07-14T11:29:12.095 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.095 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.096 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.096 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:12.096 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.096 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.096 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.097 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2023-07-14T11:29:12.098 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2023-07-14T11:29:12.098 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.098 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.098 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.098 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:12.099 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.099 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.099 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.099 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.099 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.100 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.100 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.100 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.100 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2023-07-14T11:29:12.100 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2023-07-14T11:29:12.101 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.101 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.101 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.101 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:12.101 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.102 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.103 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.103 INFO:teuthology.orchestra.run.smithi040.stdout: "cross_origin_url": { 2023-07-14T11:29:12.103 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cross_origin_url", 2023-07-14T11:29:12.103 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.103 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.104 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.104 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.104 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.104 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.105 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.106 INFO:teuthology.orchestra.run.smithi040.stdout: "crt_file": { 2023-07-14T11:29:12.106 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crt_file", 2023-07-14T11:29:12.106 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.106 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.106 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.107 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.107 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.107 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.107 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.107 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.108 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.108 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.108 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.108 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.108 INFO:teuthology.orchestra.run.smithi040.stdout: "debug": { 2023-07-14T11:29:12.109 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "debug", 2023-07-14T11:29:12.109 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.109 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.109 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.109 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.110 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.110 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.110 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.110 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable/disable debug options", 2023-07-14T11:29:12.110 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.111 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.111 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.111 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.111 INFO:teuthology.orchestra.run.smithi040.stdout: "jwt_token_ttl": { 2023-07-14T11:29:12.111 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "jwt_token_ttl", 2023-07-14T11:29:12.112 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.113 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.113 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.113 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "28800", 2023-07-14T11:29:12.113 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.113 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.114 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.114 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.114 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.114 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.114 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.115 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.115 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:12.115 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:12.115 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.115 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.116 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.116 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.116 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.116 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.116 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.117 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.117 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.117 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.117 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.117 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.118 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.118 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.118 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.118 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.119 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.119 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.119 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.119 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.119 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.120 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.120 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.120 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.120 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.120 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.121 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.121 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.121 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.121 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.122 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.123 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.123 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.123 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.123 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.124 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.124 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.124 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.124 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.124 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.125 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.125 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.125 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.125 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.125 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.126 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.126 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.126 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.126 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.126 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.127 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.127 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.127 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.127 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.127 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.128 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.128 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.128 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.128 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.128 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.129 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.129 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.129 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.129 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.129 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.130 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.130 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.130 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.130 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.130 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.131 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.131 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.131 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.131 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.131 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.132 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.132 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.132 INFO:teuthology.orchestra.run.smithi040.stdout: "motd": { 2023-07-14T11:29:12.132 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "motd", 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.133 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.134 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.134 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "The message of the day", 2023-07-14T11:29:12.134 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.134 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.134 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.135 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.135 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect_resolve_ip_addr": { 2023-07-14T11:29:12.135 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "redirect_resolve_ip_addr", 2023-07-14T11:29:12.135 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.135 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.136 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.136 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.136 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.136 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.136 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.137 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.137 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.137 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.137 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.137 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.138 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:12.138 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:12.138 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.138 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.138 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.139 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:12.139 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.139 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.139 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.139 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.140 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.140 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.140 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.140 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.140 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:12.141 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:12.141 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.141 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.141 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.141 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8080", 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.142 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.143 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.143 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.143 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:12.143 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:12.143 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.144 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.144 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.144 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.144 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.144 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.145 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.145 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.145 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.145 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.145 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.146 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.146 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl_server_port": { 2023-07-14T11:29:12.146 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl_server_port", 2023-07-14T11:29:12.146 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.146 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8443", 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.147 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.148 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.148 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.148 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.148 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.149 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:12.149 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:12.149 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.149 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.149 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.150 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "redirect", 2023-07-14T11:29:12.150 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.150 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.150 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.150 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.151 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect" 2023-07-14T11:29:12.151 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.151 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.151 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.151 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.152 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.153 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.153 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:12.153 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:12.153 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:12.153 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.154 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.154 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.154 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.154 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.154 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.155 INFO:teuthology.orchestra.run.smithi040.stdout: "url_prefix": { 2023-07-14T11:29:12.155 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url_prefix", 2023-07-14T11:29:12.155 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.155 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.155 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.156 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.156 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.156 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.156 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.156 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.157 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.157 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.157 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.157 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.157 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.158 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.158 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.158 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "devicehealth", 2023-07-14T11:29:12.158 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.158 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.159 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.159 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_monitoring": { 2023-07-14T11:29:12.159 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_monitoring", 2023-07-14T11:29:12.159 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.159 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.160 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.160 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.160 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.160 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.160 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.161 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "monitor device health metrics", 2023-07-14T11:29:12.161 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.161 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.161 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.161 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.162 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.162 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.162 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.162 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.162 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.163 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.163 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.163 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.163 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.163 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.164 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.164 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.164 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.164 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.164 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.165 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.165 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.165 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.165 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.165 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.166 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.166 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.166 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.166 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.166 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.167 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.168 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.168 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.168 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.168 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.168 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.169 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.169 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.169 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.169 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.170 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.170 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.170 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.170 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.170 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.171 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.172 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.172 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.172 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.172 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.172 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.173 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.173 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.173 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.173 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.173 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.174 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.174 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.174 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.174 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.174 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: "mark_out_threshold": { 2023-07-14T11:29:12.175 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mark_out_threshold", 2023-07-14T11:29:12.176 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.176 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.176 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.176 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2419200", 2023-07-14T11:29:12.176 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.177 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.177 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.177 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically mark OSD if it may fail before this long", 2023-07-14T11:29:12.177 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_name": { 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_name", 2023-07-14T11:29:12.178 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.179 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.179 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.179 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "device_health_metrics", 2023-07-14T11:29:12.179 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.179 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.180 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.180 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name of pool in which to store device health metrics", 2023-07-14T11:29:12.180 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.180 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.181 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.181 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.181 INFO:teuthology.orchestra.run.smithi040.stdout: "retention_period": { 2023-07-14T11:29:12.181 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retention_period", 2023-07-14T11:29:12.181 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.182 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.182 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.182 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15552000", 2023-07-14T11:29:12.182 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.182 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain device health metrics", 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.183 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.184 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_frequency": { 2023-07-14T11:29:12.184 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_frequency", 2023-07-14T11:29:12.184 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.184 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.184 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.185 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:12.185 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.185 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.185 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.185 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to scrape device health metrics", 2023-07-14T11:29:12.186 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.186 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.186 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.186 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.186 INFO:teuthology.orchestra.run.smithi040.stdout: "self_heal": { 2023-07-14T11:29:12.187 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "self_heal", 2023-07-14T11:29:12.187 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.187 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.187 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.187 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.188 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.188 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.188 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.188 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "preemptively heal cluster around devices that may fail", 2023-07-14T11:29:12.188 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.189 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.189 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.189 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.189 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:12.189 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:12.190 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.190 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.190 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.190 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:12.191 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.191 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.191 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.191 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and check device health", 2023-07-14T11:29:12.191 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.192 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.192 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.192 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.192 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_threshold": { 2023-07-14T11:29:12.192 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_threshold", 2023-07-14T11:29:12.193 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.193 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.193 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.193 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7257600", 2023-07-14T11:29:12.193 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.194 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.194 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.194 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise health warning if OSD may fail before this long", 2023-07-14T11:29:12.194 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.194 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.195 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.195 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.195 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.195 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.195 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.196 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "diskprediction_local", 2023-07-14T11:29:12.196 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.196 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.196 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.197 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.197 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.197 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.197 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.197 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.198 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.198 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.198 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.198 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.198 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.199 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.199 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.199 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.199 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.199 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.200 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.200 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.200 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.200 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.200 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.201 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.201 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.201 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.201 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.201 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.202 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.202 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.202 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.202 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.202 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.203 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.203 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.203 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.203 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.203 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.204 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.204 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.204 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.204 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.204 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.205 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.205 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.205 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.205 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.205 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.206 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.206 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.206 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.206 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.206 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.207 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.207 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.207 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.207 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.207 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.208 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.208 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.208 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.208 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.208 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.209 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.209 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.209 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.209 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.210 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.211 INFO:teuthology.orchestra.run.smithi040.stdout: "predict_interval": { 2023-07-14T11:29:12.211 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predict_interval", 2023-07-14T11:29:12.211 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.211 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.212 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.212 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:12.212 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.212 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.212 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.213 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.213 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.213 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.213 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.213 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "predictor_model": { 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predictor_model", 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.214 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "prophetstor", 2023-07-14T11:29:12.215 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.215 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.215 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.215 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.215 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.216 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.216 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.216 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.216 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:12.216 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:12.217 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.217 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.217 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.217 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:12.217 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.218 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.218 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.218 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.218 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.218 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.219 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "influx", 2023-07-14T11:29:12.220 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": false, 2023-07-14T11:29:12.220 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "influxdb python module not found", 2023-07-14T11:29:12.220 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.220 INFO:teuthology.orchestra.run.smithi040.stdout: "batch_size": { 2023-07-14T11:29:12.220 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "batch_size", 2023-07-14T11:29:12.221 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.221 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.221 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.221 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5000", 2023-07-14T11:29:12.221 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.222 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.222 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.222 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2023-07-14T11:29:12.222 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.222 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.223 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.223 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.223 INFO:teuthology.orchestra.run.smithi040.stdout: "database": { 2023-07-14T11:29:12.223 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "database", 2023-07-14T11:29:12.223 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.224 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.224 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.224 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "ceph", 2023-07-14T11:29:12.224 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.224 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.225 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.225 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2023-07-14T11:29:12.225 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.225 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.225 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.226 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.226 INFO:teuthology.orchestra.run.smithi040.stdout: "hostname": { 2023-07-14T11:29:12.226 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "hostname", 2023-07-14T11:29:12.226 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.226 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.227 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server hostname", 2023-07-14T11:29:12.228 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.228 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.228 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.228 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.228 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:12.229 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:12.229 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.229 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.229 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.230 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "30", 2023-07-14T11:29:12.230 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "5", 2023-07-14T11:29:12.230 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.230 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.230 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2023-07-14T11:29:12.231 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.231 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.231 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.231 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.231 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.232 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.233 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.233 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.233 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.233 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.234 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.234 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.234 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.234 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.234 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.235 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.235 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.235 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.235 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.235 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.236 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.236 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.236 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.236 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.236 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.237 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.237 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.237 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.237 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.237 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.238 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.238 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.238 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.238 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.238 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.239 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.239 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.239 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.239 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.239 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.240 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.240 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.240 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.240 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.240 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.241 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.242 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.242 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.242 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.242 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.242 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.243 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.243 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.243 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.243 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.243 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.244 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.244 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.244 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.244 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.244 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.245 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.245 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.245 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.245 INFO:teuthology.orchestra.run.smithi040.stdout: "password": { 2023-07-14T11:29:12.245 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "password", 2023-07-14T11:29:12.246 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.246 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.246 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.246 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.246 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.247 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.247 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.247 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "password of InfluxDB server user", 2023-07-14T11:29:12.247 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.247 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.248 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.248 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.248 INFO:teuthology.orchestra.run.smithi040.stdout: "port": { 2023-07-14T11:29:12.248 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "port", 2023-07-14T11:29:12.248 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.249 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.249 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.249 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8086", 2023-07-14T11:29:12.249 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.249 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.250 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.250 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server port", 2023-07-14T11:29:12.250 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.250 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.250 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.251 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.251 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:12.251 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:12.251 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.251 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.252 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.252 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "false", 2023-07-14T11:29:12.252 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.252 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.252 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.253 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:12.253 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.253 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.253 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.253 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.254 INFO:teuthology.orchestra.run.smithi040.stdout: "threads": { 2023-07-14T11:29:12.254 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threads", 2023-07-14T11:29:12.254 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.254 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.254 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.255 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:12.255 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:12.255 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "32", 2023-07-14T11:29:12.255 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.255 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2023-07-14T11:29:12.256 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.256 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.256 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.256 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.256 INFO:teuthology.orchestra.run.smithi040.stdout: "username": { 2023-07-14T11:29:12.257 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "username", 2023-07-14T11:29:12.257 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.257 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.257 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.257 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.258 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.258 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.258 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.258 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "username of InfluxDB server user", 2023-07-14T11:29:12.258 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.259 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.259 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.259 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.259 INFO:teuthology.orchestra.run.smithi040.stdout: "verify_ssl": { 2023-07-14T11:29:12.259 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "verify_ssl", 2023-07-14T11:29:12.260 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.260 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.260 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.260 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "true", 2023-07-14T11:29:12.260 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.261 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.261 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.261 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:12.261 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.261 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.262 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.262 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.262 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.262 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.262 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "insights", 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.263 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.264 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.264 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.264 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.264 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.264 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.265 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.265 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.265 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.265 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.265 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.266 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.266 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.266 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.266 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.266 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.267 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.267 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.267 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.267 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.267 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.268 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.268 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.268 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.268 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.268 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.269 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.269 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.269 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.269 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.269 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.270 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.270 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.270 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.270 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.270 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.271 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.271 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.271 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.271 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.272 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.272 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.272 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.272 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.272 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.273 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.273 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.273 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.273 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.273 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.274 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.274 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.274 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.274 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.274 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.275 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.275 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.275 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.275 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.275 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.276 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.276 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.276 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.276 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.276 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.277 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.277 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.277 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.277 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.277 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.278 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.278 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.278 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "iostat", 2023-07-14T11:29:12.278 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.278 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.279 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.279 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.279 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.279 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.279 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.280 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.280 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.280 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.280 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.280 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.281 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.282 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.282 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.282 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.282 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.282 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.283 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.283 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.283 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.283 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.284 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.284 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.284 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.284 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.284 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.285 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.285 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.285 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.285 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.285 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.286 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.287 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.287 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.287 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.287 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.287 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.288 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.288 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.288 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.288 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.289 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.289 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.289 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.289 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.289 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.290 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.290 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.290 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.290 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.290 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.291 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.292 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.292 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.292 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.292 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.292 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.293 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.293 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.293 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.293 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "k8sevents", 2023-07-14T11:29:12.293 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.294 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.294 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.294 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_event_retention_days": { 2023-07-14T11:29:12.294 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ceph_event_retention_days", 2023-07-14T11:29:12.294 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7", 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.295 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.296 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Days to hold ceph event information within local cache", 2023-07-14T11:29:12.296 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.296 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.296 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.296 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.297 INFO:teuthology.orchestra.run.smithi040.stdout: "config_check_secs": { 2023-07-14T11:29:12.297 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_check_secs", 2023-07-14T11:29:12.297 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.297 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.297 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.298 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:12.298 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "10", 2023-07-14T11:29:12.298 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.298 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.298 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2023-07-14T11:29:12.299 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.299 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.299 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.299 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.299 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.300 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.300 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.300 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.300 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.300 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.301 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.301 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.301 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.301 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.301 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.302 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.302 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.302 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.302 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.302 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.303 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.303 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.303 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.303 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.303 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.304 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.304 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.304 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.304 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.304 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.305 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.306 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.306 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.306 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.306 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.306 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.307 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.307 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.307 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.307 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.307 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.308 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.308 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.308 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.308 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.308 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.309 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.309 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.309 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.309 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.310 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.310 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.310 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.310 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.310 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.311 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.311 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.311 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.311 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.311 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.312 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.312 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.312 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.312 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.312 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.313 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.313 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.313 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.313 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.313 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.314 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.314 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.314 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "localpool", 2023-07-14T11:29:12.314 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.314 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.315 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.315 INFO:teuthology.orchestra.run.smithi040.stdout: "failure_domain": { 2023-07-14T11:29:12.315 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "failure_domain", 2023-07-14T11:29:12.315 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.315 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.316 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.316 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "host", 2023-07-14T11:29:12.316 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.316 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.316 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.317 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "failure domain for any created local pool", 2023-07-14T11:29:12.317 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2023-07-14T11:29:12.317 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.317 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.317 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.318 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.318 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.318 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.318 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.318 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.319 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.319 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.319 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.319 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.319 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.320 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.320 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.320 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.320 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.320 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.321 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.321 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.321 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.321 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.321 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.322 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.322 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.322 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.322 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.322 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.323 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.323 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.323 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.323 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.323 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.324 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.324 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.324 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.324 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.324 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.325 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.325 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.325 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.325 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.325 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.326 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.327 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.327 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.327 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.327 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.327 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.328 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.328 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.328 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.328 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.328 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.329 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.329 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.329 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.330 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.330 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.330 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.330 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.330 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.331 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.331 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.331 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.331 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.331 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.332 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.332 INFO:teuthology.orchestra.run.smithi040.stdout: "min_size": { 2023-07-14T11:29:12.332 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_size", 2023-07-14T11:29:12.332 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.332 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.333 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default min_size for any created local pool", 2023-07-14T11:29:12.334 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2023-07-14T11:29:12.334 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.334 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.334 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.334 INFO:teuthology.orchestra.run.smithi040.stdout: "num_rep": { 2023-07-14T11:29:12.335 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "num_rep", 2023-07-14T11:29:12.335 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.335 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.335 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.335 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3", 2023-07-14T11:29:12.336 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.336 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.336 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.336 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default replica count for any created local pool", 2023-07-14T11:29:12.336 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.337 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.337 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.337 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.337 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_num": { 2023-07-14T11:29:12.337 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_num", 2023-07-14T11:29:12.338 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.338 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.338 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.338 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "128", 2023-07-14T11:29:12.338 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.339 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.339 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.339 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default pg_num for any created local pool", 2023-07-14T11:29:12.339 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.339 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.340 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.340 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.340 INFO:teuthology.orchestra.run.smithi040.stdout: "prefix": { 2023-07-14T11:29:12.340 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prefix", 2023-07-14T11:29:12.340 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.341 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.341 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.341 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.341 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.341 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.342 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.342 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name prefix for any created local pool", 2023-07-14T11:29:12.342 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.342 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.342 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.343 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.343 INFO:teuthology.orchestra.run.smithi040.stdout: "subtree": { 2023-07-14T11:29:12.343 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "subtree", 2023-07-14T11:29:12.343 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.343 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.344 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.344 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "rack", 2023-07-14T11:29:12.344 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.344 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.344 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "CRUSH level for which to create a local pool", 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.345 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.346 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.346 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.346 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mds_autoscaler", 2023-07-14T11:29:12.346 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.346 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.347 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.347 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.347 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.347 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.348 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.348 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.348 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.348 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.348 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.349 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.349 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.349 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.349 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.349 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.350 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.350 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.350 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.350 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.350 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.351 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.351 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.351 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.351 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.351 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.352 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.352 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.352 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.352 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.352 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.353 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.353 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.353 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.353 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.353 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.354 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.354 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.354 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.354 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.354 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.355 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.355 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.355 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.355 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.355 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.356 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.356 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.356 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.356 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.356 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.357 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.357 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.357 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.357 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.357 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.358 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.359 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.359 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.359 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.359 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.359 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.360 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.360 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.360 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.360 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.360 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.361 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.361 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.361 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.361 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.361 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirroring", 2023-07-14T11:29:12.362 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.362 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.362 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.362 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.362 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.363 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.364 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.364 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.364 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.364 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.365 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.366 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.366 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.366 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.366 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.367 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.367 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.367 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.367 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.367 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.368 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.368 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.368 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.368 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.368 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.369 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.369 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.372 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.372 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.373 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.373 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.373 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.373 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.374 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.375 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.375 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.375 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.375 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.375 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.376 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.376 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.376 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.376 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.376 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.377 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.377 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.377 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.377 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.377 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.378 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.378 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.378 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.378 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.378 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.379 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.379 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.379 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.379 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "nfs", 2023-07-14T11:29:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.381 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.381 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.381 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.381 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.381 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.382 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.382 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.382 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.382 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.382 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.383 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.383 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.383 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.383 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.383 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.384 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.384 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.384 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.384 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.384 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.385 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.385 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.385 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.385 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.385 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.386 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.386 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.386 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.386 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.386 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.387 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.387 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.387 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.387 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.387 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.388 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.388 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.388 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.388 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.388 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.389 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.389 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.389 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.389 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.390 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.390 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.390 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.390 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.390 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.391 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.391 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.391 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.391 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.391 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.392 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.392 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.392 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.392 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.393 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.393 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.393 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.393 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.393 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.394 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.394 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.394 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.394 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.395 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.395 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.395 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.395 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.395 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.396 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.396 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.396 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.396 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:12.396 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.397 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.397 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.397 INFO:teuthology.orchestra.run.smithi040.stdout: "fail_fs": { 2023-07-14T11:29:12.397 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "fail_fs", 2023-07-14T11:29:12.397 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.398 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.398 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.398 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.398 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.398 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.399 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.399 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2023-07-14T11:29:12.399 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.399 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.399 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.400 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.400 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.400 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.400 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.400 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.401 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.401 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.401 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.401 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.401 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.402 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.402 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.402 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.402 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.402 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.403 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.403 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.403 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.403 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.403 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.404 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.404 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.404 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.404 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.404 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.405 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.406 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.406 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.406 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.406 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.406 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.407 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.407 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.407 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.407 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.407 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.408 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.408 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.408 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.408 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.408 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.409 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.409 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.409 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.409 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.409 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.410 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.410 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.410 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.410 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.410 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.411 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.411 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.411 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.411 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.411 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.412 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.412 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.412 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.412 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.412 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.413 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.413 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.413 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.413 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.414 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.414 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator": { 2023-07-14T11:29:12.414 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:12.414 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.414 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.415 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm", 2023-07-14T11:29:12.416 INFO:teuthology.orchestra.run.smithi040.stdout: "rook", 2023-07-14T11:29:12.416 INFO:teuthology.orchestra.run.smithi040.stdout: "test_orchestrator" 2023-07-14T11:29:12.416 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.416 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Orchestrator backend", 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.417 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.418 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.418 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_perf_query", 2023-07-14T11:29:12.418 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.418 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.418 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.419 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.419 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.419 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.419 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.419 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.420 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.420 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.420 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.420 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.420 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.421 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.421 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.421 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.421 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.421 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.422 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.422 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.422 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.422 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.422 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.423 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.424 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.424 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.424 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.424 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.424 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.425 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.425 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.425 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.425 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.425 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.426 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.426 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.426 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.426 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.426 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.427 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.427 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.427 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.427 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.427 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.428 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.428 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.428 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.428 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.428 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.429 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.429 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.429 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.429 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.430 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.430 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.430 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.430 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.430 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.431 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.431 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.431 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.431 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.431 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.432 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.432 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.432 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.432 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.432 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.433 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.433 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.433 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.433 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_support", 2023-07-14T11:29:12.433 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.434 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.434 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.434 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.434 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.434 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.435 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.436 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.436 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.436 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.436 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.436 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.437 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.437 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.437 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.437 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.437 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.438 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.438 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.438 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.438 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.438 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.439 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.439 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.439 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.439 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.439 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.440 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.440 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.440 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.440 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.440 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.441 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.441 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.441 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.441 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.441 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.442 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.442 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.442 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.442 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.442 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.443 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.443 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.443 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.443 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.443 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.444 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.444 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.444 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.444 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.444 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.445 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.445 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.445 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.445 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.445 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.446 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.446 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.446 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.446 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.446 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.447 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.447 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.447 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.447 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.447 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.448 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.448 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.448 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.448 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.448 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_autoscaler", 2023-07-14T11:29:12.449 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.449 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.449 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.449 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.449 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.450 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.450 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.450 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.450 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.451 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.451 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.451 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.451 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.451 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.452 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.452 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.452 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.452 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.452 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.453 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.453 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.453 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.453 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.453 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.454 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.454 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.454 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.454 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.454 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.455 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.455 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.455 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.455 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.455 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.456 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.456 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.456 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.456 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.456 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.457 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.457 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.457 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.457 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.457 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.458 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.458 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.458 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.458 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.458 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.459 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.460 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.460 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.460 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.460 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.461 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.461 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.461 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.461 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.461 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.462 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.462 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.462 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.462 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.462 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.463 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.463 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.463 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.463 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.463 INFO:teuthology.orchestra.run.smithi040.stdout: "noautoscale": { 2023-07-14T11:29:12.464 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "noautoscale", 2023-07-14T11:29:12.464 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.464 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.464 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.465 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.465 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.465 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.465 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.465 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "global autoscale flag", 2023-07-14T11:29:12.466 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Option to turn on/off the autoscaler for all pools", 2023-07-14T11:29:12.466 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.466 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.466 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.466 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:12.467 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.468 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.468 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.468 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.468 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.469 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.469 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.469 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.469 INFO:teuthology.orchestra.run.smithi040.stdout: "threshold": { 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threshold", 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:12.470 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1.0", 2023-07-14T11:29:12.471 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.471 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.471 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "scaling threshold", 2023-07-14T11:29:12.471 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2023-07-14T11:29:12.471 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.472 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.472 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.472 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.472 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.472 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.473 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "progress", 2023-07-14T11:29:12.473 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.473 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.473 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.473 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_pg_recovery_event": { 2023-07-14T11:29:12.474 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_pg_recovery_event", 2023-07-14T11:29:12.474 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.474 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.474 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.474 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.475 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.475 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.475 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.475 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow the module to show pg recovery progress", 2023-07-14T11:29:12.475 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.476 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.476 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.476 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.476 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:12.476 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:12.477 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.477 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.477 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.477 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.477 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.478 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.478 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.478 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.478 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.480 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.480 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.480 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.480 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.481 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.481 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.481 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.481 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.481 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.482 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.482 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.482 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.482 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.482 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.483 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.483 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.483 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.483 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.484 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.484 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.484 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.484 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.484 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.485 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.485 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.485 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.485 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.485 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.486 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.486 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.486 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.486 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.487 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.487 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.487 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.487 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.487 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.488 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.488 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.488 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.488 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.488 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.489 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.489 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.489 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.489 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.489 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.490 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.490 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.490 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.490 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.490 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.491 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.491 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.491 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.491 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.491 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.492 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.492 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.492 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.492 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.492 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.493 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.493 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.493 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.493 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.493 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.494 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.494 INFO:teuthology.orchestra.run.smithi040.stdout: "max_completed_events": { 2023-07-14T11:29:12.494 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_completed_events", 2023-07-14T11:29:12.494 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.494 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.495 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.495 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "50", 2023-07-14T11:29:12.495 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.495 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.495 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.496 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "number of past completed events to remember", 2023-07-14T11:29:12.496 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.496 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.496 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.496 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.497 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:12.497 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:12.497 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.497 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.497 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.498 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:12.498 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.498 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.498 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.498 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long the module is going to sleep", 2023-07-14T11:29:12.499 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.499 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.499 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.499 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.499 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus", 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.500 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.501 INFO:teuthology.orchestra.run.smithi040.stdout: "cache": { 2023-07-14T11:29:12.501 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cache", 2023-07-14T11:29:12.501 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.501 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.501 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.502 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.502 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.502 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.502 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.502 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.503 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.503 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.503 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.503 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.503 INFO:teuthology.orchestra.run.smithi040.stdout: "exclude_perf_counters": { 2023-07-14T11:29:12.504 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "exclude_perf_counters", 2023-07-14T11:29:12.504 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.504 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.504 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.504 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.505 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.505 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.505 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.505 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Do not include perf-counters in the metrics output", 2023-07-14T11:29:12.505 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2023-07-14T11:29:12.506 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.506 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.506 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.506 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.506 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.507 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.507 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.507 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.507 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.507 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.508 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.508 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.508 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.508 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.508 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.509 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.509 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.509 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.509 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.509 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.510 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.511 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.511 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.511 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.511 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.512 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.512 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.512 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.512 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.513 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.513 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.513 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.513 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.513 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.514 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.514 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.514 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.514 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.514 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.515 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.515 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.515 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.515 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.515 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.516 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.516 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.516 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.516 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.517 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.517 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.517 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.517 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.517 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.518 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.518 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.518 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.518 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.518 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.519 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.519 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.519 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.519 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.519 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.520 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.520 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.520 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.520 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.521 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.521 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools": { 2023-07-14T11:29:12.521 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools", 2023-07-14T11:29:12.521 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.521 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.522 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.523 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.523 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.523 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.523 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.523 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools_refresh_interval": { 2023-07-14T11:29:12.524 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools_refresh_interval", 2023-07-14T11:29:12.524 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.524 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.524 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.524 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300", 2023-07-14T11:29:12.525 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.525 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.525 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.525 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.525 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_interval": { 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_interval", 2023-07-14T11:29:12.526 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.527 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.527 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.527 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15.0", 2023-07-14T11:29:12.527 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.527 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.528 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.528 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.528 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.528 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.529 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.530 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.530 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:12.530 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.530 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.531 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.531 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2023-07-14T11:29:12.531 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.531 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.531 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.532 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.532 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:12.532 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:12.532 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.532 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "9283", 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.533 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the port on which the module listens for HTTP requests", 2023-07-14T11:29:12.534 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.534 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.534 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.534 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.534 INFO:teuthology.orchestra.run.smithi040.stdout: "stale_cache_strategy": { 2023-07-14T11:29:12.535 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stale_cache_strategy", 2023-07-14T11:29:12.535 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.535 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.535 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.535 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "log", 2023-07-14T11:29:12.536 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.536 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.536 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.536 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.536 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.537 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.537 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.537 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.537 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:12.537 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:12.538 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.538 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.538 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.538 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "default", 2023-07-14T11:29:12.538 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.539 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.539 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.539 INFO:teuthology.orchestra.run.smithi040.stdout: "default", 2023-07-14T11:29:12.539 INFO:teuthology.orchestra.run.smithi040.stdout: "error" 2023-07-14T11:29:12.539 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.540 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.540 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.540 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.540 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.540 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.541 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:12.541 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:12.541 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.541 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.541 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.542 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:12.542 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:12.542 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:12.542 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.542 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.543 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.543 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.543 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.543 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.543 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.544 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.544 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.544 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_support", 2023-07-14T11:29:12.544 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.544 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.545 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.545 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.545 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.545 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.545 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.546 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.546 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.546 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.546 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.546 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.547 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.547 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.547 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.547 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.547 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.548 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.548 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.548 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.548 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.548 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.549 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.549 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.549 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.549 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.549 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.550 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.551 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.551 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.551 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.551 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.551 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.552 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.552 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.552 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.552 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.552 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.553 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.553 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.553 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.553 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.553 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.554 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.554 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.554 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.554 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.554 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.555 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.555 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.555 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.555 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.555 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.556 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.556 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.556 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.556 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.556 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.557 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.557 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.557 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.557 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.557 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.558 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.558 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.558 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.558 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.558 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_snap_create": { 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_snap_create", 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.559 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:12.560 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.560 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.560 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.560 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.560 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.561 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.561 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.561 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.561 INFO:teuthology.orchestra.run.smithi040.stdout: "mirror_snapshot_schedule": { 2023-07-14T11:29:12.561 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirror_snapshot_schedule", 2023-07-14T11:29:12.562 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.562 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.562 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.562 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.562 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.563 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.563 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.563 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.563 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.563 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.564 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.564 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.564 INFO:teuthology.orchestra.run.smithi040.stdout: "trash_purge_schedule": { 2023-07-14T11:29:12.564 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "trash_purge_schedule", 2023-07-14T11:29:12.564 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.565 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.565 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.565 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.565 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.565 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.566 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.566 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.566 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.566 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.566 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.567 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.567 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.567 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.567 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.568 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "restful", 2023-07-14T11:29:12.568 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.568 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.568 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.568 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_auth": { 2023-07-14T11:29:12.569 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_auth", 2023-07-14T11:29:12.569 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.569 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.569 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.570 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.570 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.570 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.570 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.570 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.571 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.571 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.571 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.571 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.572 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:12.572 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:12.572 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.572 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.572 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.573 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.573 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.573 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.573 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.573 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.574 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.574 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.574 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.574 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.574 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.575 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.575 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.575 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.575 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.575 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.576 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.576 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.576 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.576 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.576 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.577 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.578 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.578 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.578 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.578 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.578 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.579 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.579 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.579 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.579 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.579 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.580 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.580 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.580 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.580 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.580 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.581 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.582 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.582 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.582 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.584 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.584 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.584 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.584 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.584 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.585 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.585 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.585 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.585 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.585 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.586 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.586 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.586 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.586 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.586 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.587 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.587 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.587 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.587 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.587 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.588 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.588 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.588 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.588 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.588 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.589 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.590 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.590 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.590 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.590 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.590 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.591 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.591 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.591 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:12.591 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:12.591 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.592 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.592 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.592 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.592 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.592 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.593 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.593 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.593 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.593 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.593 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.594 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.594 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.594 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.594 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.594 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rgw", 2023-07-14T11:29:12.595 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.595 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.595 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.595 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.595 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.596 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.596 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.596 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.596 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.596 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.597 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.597 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.597 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.597 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.597 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.598 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.599 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.599 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.599 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.599 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.599 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.600 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.600 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.600 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.600 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.600 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.601 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.601 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.601 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.601 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.601 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.602 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.602 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.602 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.602 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.602 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.603 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.603 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.603 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.603 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.603 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.604 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.604 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.604 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.604 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.604 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.605 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.606 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.607 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.607 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.607 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.607 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.607 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.608 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.608 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.608 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.608 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.608 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.609 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rook", 2023-07-14T11:29:12.610 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.610 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.610 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.610 INFO:teuthology.orchestra.run.smithi040.stdout: "drive_group_interval": { 2023-07-14T11:29:12.611 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "drive_group_interval", 2023-07-14T11:29:12.611 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.611 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.611 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.611 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300.0", 2023-07-14T11:29:12.612 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.612 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.612 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.612 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval in seconds between re-application of applied drive_groups", 2023-07-14T11:29:12.612 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.613 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.614 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.614 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.614 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.614 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.614 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.615 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.615 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.615 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.615 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.615 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.616 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.616 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.616 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.616 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.617 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.617 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.617 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.617 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.617 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.618 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.618 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.618 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.618 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.618 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.619 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.619 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.619 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.619 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.619 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.620 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.620 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.620 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.620 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.620 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.621 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.621 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.621 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.621 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.622 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.622 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.622 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.622 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.622 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.623 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.624 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.624 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.624 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.624 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.624 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.625 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.625 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.625 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.625 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.625 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.626 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.626 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.626 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.626 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.626 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.627 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.627 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.627 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.627 INFO:teuthology.orchestra.run.smithi040.stdout: "storage_class": { 2023-07-14T11:29:12.627 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "storage_class", 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "local", 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.628 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.629 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.629 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "storage class name for LSO-discovered PVs", 2023-07-14T11:29:12.629 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.629 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.629 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.630 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.630 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.630 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.630 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.630 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "selftest", 2023-07-14T11:29:12.631 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.631 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.631 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.631 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.631 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.632 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.632 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.632 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.632 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.632 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.633 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.634 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.634 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.634 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.634 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.634 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.635 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.635 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.635 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.635 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.635 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.636 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.636 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.636 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.636 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.636 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.637 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.637 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.637 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.637 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.637 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.638 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.638 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.638 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.638 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.638 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.639 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.640 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.640 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.640 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.640 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.641 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.642 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.642 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.642 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.642 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.642 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.643 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.643 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.643 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.643 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.643 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.644 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.644 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.644 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.644 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: "roption1": { 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption1", 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.645 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.646 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.646 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.646 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.646 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.646 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.647 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.647 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.647 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.647 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.647 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.648 INFO:teuthology.orchestra.run.smithi040.stdout: "roption2": { 2023-07-14T11:29:12.648 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption2", 2023-07-14T11:29:12.648 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.648 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.648 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.649 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "xyz", 2023-07-14T11:29:12.649 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.649 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.649 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.649 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.650 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.650 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.650 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.650 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.650 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption1": { 2023-07-14T11:29:12.651 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption1", 2023-07-14T11:29:12.651 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.651 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.651 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.651 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.652 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.652 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.652 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.652 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.652 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption2": { 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption2", 2023-07-14T11:29:12.653 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.654 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.654 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.654 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.654 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.654 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.655 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.655 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.655 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.655 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.655 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.656 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.656 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption3": { 2023-07-14T11:29:12.656 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption3", 2023-07-14T11:29:12.656 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.656 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.657 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.657 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.657 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.657 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.657 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.658 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.658 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.658 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.658 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.658 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.659 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption4": { 2023-07-14T11:29:12.659 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption4", 2023-07-14T11:29:12.659 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.659 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.659 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.660 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.660 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.660 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.660 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.660 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.661 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.661 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.661 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.661 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.661 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption5": { 2023-07-14T11:29:12.662 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption5", 2023-07-14T11:29:12.662 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.662 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.662 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.662 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.663 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.663 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.663 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.663 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.663 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.664 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.664 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.664 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.664 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption6": { 2023-07-14T11:29:12.664 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption6", 2023-07-14T11:29:12.665 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.665 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.665 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.665 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.665 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.666 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.666 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.666 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.666 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.666 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.667 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.667 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.667 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption7": { 2023-07-14T11:29:12.667 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption7", 2023-07-14T11:29:12.667 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.668 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.668 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.668 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.668 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:12.669 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "42", 2023-07-14T11:29:12.669 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.669 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.669 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.670 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.670 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.670 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.670 INFO:teuthology.orchestra.run.smithi040.stdout: "testkey": { 2023-07-14T11:29:12.670 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testkey", 2023-07-14T11:29:12.671 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.671 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.671 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.671 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.671 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.672 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.672 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.672 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.672 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.672 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.673 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.673 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.673 INFO:teuthology.orchestra.run.smithi040.stdout: "testlkey": { 2023-07-14T11:29:12.673 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testlkey", 2023-07-14T11:29:12.673 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.674 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.674 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.674 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.674 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.675 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.676 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.676 INFO:teuthology.orchestra.run.smithi040.stdout: "testnewline": { 2023-07-14T11:29:12.676 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testnewline", 2023-07-14T11:29:12.676 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.676 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.677 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.677 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.677 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.677 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.677 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.678 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.678 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.678 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.678 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.678 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.679 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.679 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.679 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.679 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snap_schedule", 2023-07-14T11:29:12.679 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.680 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.680 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.680 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_m_granularity": { 2023-07-14T11:29:12.680 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_m_granularity", 2023-07-14T11:29:12.680 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.681 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.681 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.681 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.681 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.681 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.682 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.682 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow minute scheduled snapshots", 2023-07-14T11:29:12.682 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.682 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.682 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.683 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.683 INFO:teuthology.orchestra.run.smithi040.stdout: "dump_on_update": { 2023-07-14T11:29:12.683 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dump_on_update", 2023-07-14T11:29:12.683 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.683 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.684 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.684 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.684 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.684 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.684 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "dump database to debug log on update", 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.685 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.686 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.686 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.686 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.686 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.686 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.687 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.687 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.687 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.687 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.687 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.688 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.688 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.688 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.688 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.688 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.689 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.690 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.690 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.690 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.690 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.690 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.691 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.691 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.691 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.691 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.691 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.692 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.693 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.693 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.693 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.693 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.693 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.694 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.694 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.694 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.694 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.694 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.695 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.695 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.695 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.695 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.695 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.696 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.697 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.697 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.697 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.697 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.698 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.698 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.698 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.698 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.698 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.699 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.699 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.699 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.699 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.699 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.700 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stats", 2023-07-14T11:29:12.700 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.700 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.700 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.700 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.701 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.701 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.701 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.701 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.701 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.702 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.703 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.703 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.703 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.703 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.703 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.704 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.704 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.704 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.704 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.704 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.705 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.705 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.705 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.705 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.705 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.706 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.706 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.706 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.706 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.706 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.707 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.707 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.707 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.707 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.707 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.708 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.708 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.708 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.708 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.708 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.709 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.710 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.710 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.710 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.710 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.710 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.711 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.711 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.711 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.711 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.711 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.712 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.712 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.712 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.712 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.713 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.713 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.713 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.713 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.713 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.714 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.714 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.714 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.714 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.714 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.715 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.715 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "status", 2023-07-14T11:29:12.715 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.715 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.715 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.716 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.717 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.717 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.717 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.717 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.718 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.718 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.718 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.718 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.718 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.719 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.719 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.719 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.719 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.719 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.720 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.721 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.721 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.721 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.721 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.721 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.722 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.722 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.722 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.722 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.722 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.723 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.723 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.723 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.723 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.723 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.724 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.724 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.724 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.724 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.724 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.725 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.725 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.725 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.725 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.725 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.726 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.726 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.726 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.726 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.726 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.727 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.728 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.728 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.728 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.728 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.728 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.729 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.729 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.729 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.729 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.729 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.730 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.730 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telegraf", 2023-07-14T11:29:12.730 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.730 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.730 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.731 INFO:teuthology.orchestra.run.smithi040.stdout: "address": { 2023-07-14T11:29:12.731 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "address", 2023-07-14T11:29:12.731 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.731 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.731 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.732 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2023-07-14T11:29:12.732 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.732 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.732 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.732 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.733 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.733 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.733 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.733 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.733 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:12.734 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:12.734 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.734 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.734 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.735 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15", 2023-07-14T11:29:12.735 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.735 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.735 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.735 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.736 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.736 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.736 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.736 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.736 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.737 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.737 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.737 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.737 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.737 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.738 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.738 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.738 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.738 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.738 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.739 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.739 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.739 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.739 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.739 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.740 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.740 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.740 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.740 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.740 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.741 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.741 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.741 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.741 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.741 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.742 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.742 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.742 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.742 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.743 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.743 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.743 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.743 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.743 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.744 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.744 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.744 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.744 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.745 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.746 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.746 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.746 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.746 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.746 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.747 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.747 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.747 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.747 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.747 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.748 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.748 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.748 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.748 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.748 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.749 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.749 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.749 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.749 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.749 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.750 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.750 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.750 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.750 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.750 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.751 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.751 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.751 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.751 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telemetry", 2023-07-14T11:29:12.751 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.752 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.752 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.752 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_basic": { 2023-07-14T11:29:12.752 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_basic", 2023-07-14T11:29:12.752 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.753 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.754 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share basic cluster information (size, version)", 2023-07-14T11:29:12.754 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.754 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.754 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.754 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.755 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_crash": { 2023-07-14T11:29:12.755 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_crash", 2023-07-14T11:29:12.755 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.755 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.755 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.756 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.756 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.756 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.756 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.757 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2023-07-14T11:29:12.757 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.757 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.757 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.758 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.758 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_device": { 2023-07-14T11:29:12.758 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_device", 2023-07-14T11:29:12.758 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.758 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.759 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2023-07-14T11:29:12.760 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.760 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.760 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.760 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.761 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_ident": { 2023-07-14T11:29:12.761 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_ident", 2023-07-14T11:29:12.761 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.761 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.761 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.762 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.762 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.762 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.762 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.762 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2023-07-14T11:29:12.763 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.763 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.763 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.763 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.763 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_perf": { 2023-07-14T11:29:12.764 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_perf", 2023-07-14T11:29:12.764 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.764 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.764 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.764 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.765 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.765 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.765 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.765 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share various performance metrics of a cluster", 2023-07-14T11:29:12.766 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.766 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.766 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.766 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.766 INFO:teuthology.orchestra.run.smithi040.stdout: "contact": { 2023-07-14T11:29:12.767 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "contact", 2023-07-14T11:29:12.767 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.767 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.767 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.767 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.768 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.768 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.768 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.768 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.769 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.769 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.769 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.769 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.769 INFO:teuthology.orchestra.run.smithi040.stdout: "description": { 2023-07-14T11:29:12.770 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "description", 2023-07-14T11:29:12.770 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.770 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.770 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.770 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.771 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.772 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.772 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.772 INFO:teuthology.orchestra.run.smithi040.stdout: "device_url": { 2023-07-14T11:29:12.772 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_url", 2023-07-14T11:29:12.773 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.773 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.773 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.773 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/device", 2023-07-14T11:29:12.773 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.774 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.774 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.774 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.774 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.774 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.775 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.775 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.775 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:12.775 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:12.775 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.776 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.776 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.776 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.776 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.777 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.777 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.777 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.777 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.777 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.778 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.778 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.778 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:12.778 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:12.778 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.779 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.779 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.779 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "24", 2023-07-14T11:29:12.779 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "8", 2023-07-14T11:29:12.779 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.780 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.780 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.780 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.780 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.781 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.781 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.781 INFO:teuthology.orchestra.run.smithi040.stdout: "last_opt_revision": { 2023-07-14T11:29:12.781 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "last_opt_revision", 2023-07-14T11:29:12.781 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.782 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.782 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.782 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1", 2023-07-14T11:29:12.782 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.782 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.783 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.783 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.783 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.783 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.784 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.784 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.784 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard": { 2023-07-14T11:29:12.784 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard", 2023-07-14T11:29:12.785 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.785 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.785 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.785 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.785 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.786 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.786 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.786 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.786 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.786 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard_description": { 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard_description", 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.787 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.788 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.788 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.788 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.788 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.789 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.790 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.790 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.790 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.790 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.790 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.791 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.791 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.791 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.791 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.791 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.792 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.792 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.792 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.792 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.792 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.793 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.793 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.793 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.793 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.793 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.794 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.795 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.795 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.795 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.795 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.795 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.796 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.796 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.796 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.796 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.797 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.797 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.797 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.797 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.797 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.798 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.798 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.798 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.798 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.798 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.799 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.799 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.799 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.799 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.799 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.800 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.800 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.800 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.800 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.800 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.801 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.801 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.801 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.801 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.801 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.803 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.803 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.803 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.803 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.803 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.804 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.804 INFO:teuthology.orchestra.run.smithi040.stdout: "organization": { 2023-07-14T11:29:12.804 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "organization", 2023-07-14T11:29:12.804 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.804 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.805 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.806 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.806 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.806 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.806 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.806 INFO:teuthology.orchestra.run.smithi040.stdout: "proxy": { 2023-07-14T11:29:12.807 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "proxy", 2023-07-14T11:29:12.807 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.807 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.807 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.807 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.808 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.808 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.808 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.808 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.808 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.809 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.809 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.809 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.809 INFO:teuthology.orchestra.run.smithi040.stdout: "url": { 2023-07-14T11:29:12.809 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url", 2023-07-14T11:29:12.810 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.810 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.810 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.810 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/report", 2023-07-14T11:29:12.810 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.811 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.811 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.811 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.811 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.811 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.812 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.812 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.812 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.812 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.812 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.813 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "test_orchestrator", 2023-07-14T11:29:12.813 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.813 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.813 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.813 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.814 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.814 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.814 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.814 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.814 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.815 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.815 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.815 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.815 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.816 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.816 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.816 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.816 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.817 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.817 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.817 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.817 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.817 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.818 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.818 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.818 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.818 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.818 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.819 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.819 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.819 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.819 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.819 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.820 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.820 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.820 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.820 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.820 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.821 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.821 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.821 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.821 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.821 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.822 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.822 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.822 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.822 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.822 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.823 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.823 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.823 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.823 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.823 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.824 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.824 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.824 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.824 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.824 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.825 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.825 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.825 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.825 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.825 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.826 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.827 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.827 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.827 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.827 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.827 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.828 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.828 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.828 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.828 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "volumes", 2023-07-14T11:29:12.828 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.829 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.829 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.829 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.829 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.829 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.830 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.830 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.830 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.830 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.830 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.831 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.831 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.831 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.831 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.831 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.832 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.832 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.832 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.832 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.832 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.833 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.833 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.833 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.833 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.833 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.834 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.834 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.834 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.834 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.834 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.835 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.835 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.835 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.835 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.835 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.836 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.836 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.836 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.836 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.836 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.837 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.837 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.837 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.837 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.837 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.838 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.838 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.838 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.838 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.839 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.839 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.839 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.839 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.839 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.840 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.840 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.840 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.840 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.840 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.841 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.841 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.841 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.841 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.841 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.842 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.842 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.842 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.842 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.842 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.843 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.843 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.843 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.843 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_clones": { 2023-07-14T11:29:12.843 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_clones", 2023-07-14T11:29:12.844 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.844 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.844 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.844 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4", 2023-07-14T11:29:12.844 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.845 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.845 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.845 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Number of asynchronous cloner threads", 2023-07-14T11:29:12.845 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.846 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.846 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.846 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.846 INFO:teuthology.orchestra.run.smithi040.stdout: "snapshot_clone_delay": { 2023-07-14T11:29:12.846 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snapshot_clone_delay", 2023-07-14T11:29:12.847 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.847 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.847 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.847 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:12.847 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.848 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.848 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.848 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2023-07-14T11:29:12.848 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.849 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.849 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.849 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.849 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.849 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.850 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.850 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix", 2023-07-14T11:29:12.850 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.850 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.850 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.851 INFO:teuthology.orchestra.run.smithi040.stdout: "discovery_interval": { 2023-07-14T11:29:12.851 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "discovery_interval", 2023-07-14T11:29:12.851 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:12.851 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.851 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.852 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "100", 2023-07-14T11:29:12.852 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.852 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.852 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.852 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.853 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.853 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.853 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.853 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.853 INFO:teuthology.orchestra.run.smithi040.stdout: "identifier": { 2023-07-14T11:29:12.854 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "identifier", 2023-07-14T11:29:12.854 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.854 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.854 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.854 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.855 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.855 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.855 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.855 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.855 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.856 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.856 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.856 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.856 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:12.856 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:12.857 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.857 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.857 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.857 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:12.858 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.858 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.858 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.858 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.858 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.859 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.859 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.859 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.859 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.859 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.860 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.861 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.861 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.861 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.861 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.861 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.862 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.862 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.862 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.862 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.862 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.863 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.863 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.863 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.863 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.863 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.864 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.864 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.864 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.864 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.864 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.865 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.865 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.865 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.865 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.865 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.866 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.867 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.867 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.867 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.867 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.867 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.868 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.868 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.868 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.868 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.868 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.869 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.869 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.869 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.869 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.870 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.871 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.871 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.871 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.871 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.871 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.872 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.872 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.872 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.872 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.872 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.873 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.873 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.873 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_host": { 2023-07-14T11:29:12.873 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_host", 2023-07-14T11:29:12.873 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.874 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.874 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.874 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.874 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.874 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.875 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.876 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_port": { 2023-07-14T11:29:12.876 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_port", 2023-07-14T11:29:12.876 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.876 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10051", 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.877 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.878 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.878 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.878 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.878 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.878 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_sender": { 2023-07-14T11:29:12.879 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_sender", 2023-07-14T11:29:12.879 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.879 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.879 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.879 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/usr/bin/zabbix_sender", 2023-07-14T11:29:12.880 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.880 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.880 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.880 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.880 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.881 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.881 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.881 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.881 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.881 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.882 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:12.882 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.882 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.882 INFO:teuthology.orchestra.run.smithi040.stdout: "gid": 25571, 2023-07-14T11:29:12.882 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "y", 2023-07-14T11:29:12.883 INFO:teuthology.orchestra.run.smithi040.stdout: "mgr_features": 4540138322906710015, 2023-07-14T11:29:12.883 INFO:teuthology.orchestra.run.smithi040.stdout: "available_modules": [ 2023-07-14T11:29:12.883 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.883 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alerts", 2023-07-14T11:29:12.883 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.884 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.884 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.884 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:12.884 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:12.884 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.885 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.885 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.885 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:12.885 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.885 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.886 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.886 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How frequently to reexamine health status", 2023-07-14T11:29:12.886 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.886 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.886 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.887 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.888 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.888 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.888 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.888 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.888 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.889 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.889 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.889 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.889 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.890 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.891 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.891 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.891 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.891 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.891 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.892 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.892 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.892 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.892 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.892 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.893 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.893 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.893 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.893 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.893 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.894 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.894 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.894 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.894 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.894 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.895 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.895 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.895 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.895 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.895 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.896 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.896 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.896 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.896 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.896 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.897 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.897 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.897 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.897 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.897 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.898 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.898 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.898 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.898 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.898 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.899 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.899 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.899 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.899 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.899 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.900 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.900 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.900 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.900 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.900 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.901 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_destination": { 2023-07-14T11:29:12.901 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_destination", 2023-07-14T11:29:12.901 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.901 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.901 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.902 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.902 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.902 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.902 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.902 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email address to send alerts to", 2023-07-14T11:29:12.903 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.903 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.903 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.903 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.903 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_from_name": { 2023-07-14T11:29:12.904 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_from_name", 2023-07-14T11:29:12.904 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.904 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.904 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.904 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "Ceph", 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email From: name", 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.905 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.906 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.906 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.906 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_host": { 2023-07-14T11:29:12.906 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_host", 2023-07-14T11:29:12.906 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.907 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.907 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.907 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.907 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.907 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.908 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.908 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP server", 2023-07-14T11:29:12.908 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.908 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.908 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.909 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.909 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_password": { 2023-07-14T11:29:12.909 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_password", 2023-07-14T11:29:12.909 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.909 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.910 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Password to authenticate with", 2023-07-14T11:29:12.911 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.911 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.911 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.911 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.912 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_port": { 2023-07-14T11:29:12.912 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_port", 2023-07-14T11:29:12.912 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.912 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.913 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.913 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "465", 2023-07-14T11:29:12.913 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.913 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.913 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP port", 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.914 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_sender": { 2023-07-14T11:29:12.915 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_sender", 2023-07-14T11:29:12.915 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.915 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.915 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.915 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.916 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.916 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.916 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.916 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP envelope sender", 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_ssl": { 2023-07-14T11:29:12.917 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_ssl", 2023-07-14T11:29:12.918 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.918 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.918 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.918 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.918 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.919 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.919 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.919 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use SSL to connect to SMTP server", 2023-07-14T11:29:12.919 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.919 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.920 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.920 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.920 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_user": { 2023-07-14T11:29:12.920 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_user", 2023-07-14T11:29:12.921 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.921 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.921 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.921 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.921 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.922 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.922 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.922 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "User to authenticate as", 2023-07-14T11:29:12.922 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.922 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.923 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "balancer", 2023-07-14T11:29:12.924 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.924 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.924 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.924 INFO:teuthology.orchestra.run.smithi040.stdout: "active": { 2023-07-14T11:29:12.924 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "active", 2023-07-14T11:29:12.925 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.925 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.925 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.925 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically balance PGs across cluster", 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.926 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.927 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.927 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.927 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_time": { 2023-07-14T11:29:12.927 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_time", 2023-07-14T11:29:12.927 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.928 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.928 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.928 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0000", 2023-07-14T11:29:12.928 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.928 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.929 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.929 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "beginning time of day to automatically balance", 2023-07-14T11:29:12.929 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:12.929 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.929 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.930 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.930 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_weekday": { 2023-07-14T11:29:12.930 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_weekday", 2023-07-14T11:29:12.930 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:12.931 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.931 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.931 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:12.931 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:12.931 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.932 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.933 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_max_iterations": { 2023-07-14T11:29:12.933 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_max_iterations", 2023-07-14T11:29:12.933 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:12.933 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.934 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.934 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "25", 2023-07-14T11:29:12.934 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:12.934 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "250", 2023-07-14T11:29:12.934 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.935 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum number of iterations to attempt optimization", 2023-07-14T11:29:12.935 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.935 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.935 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.935 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.936 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_metrics": { 2023-07-14T11:29:12.936 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_metrics", 2023-07-14T11:29:12.936 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.936 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.936 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.937 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "pgs,objects,bytes", 2023-07-14T11:29:12.937 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.937 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.937 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.937 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "metrics with which to calculate OSD utilization", 2023-07-14T11:29:12.938 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2023-07-14T11:29:12.938 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.938 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.938 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.938 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_step": { 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_step", 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.5", 2023-07-14T11:29:12.939 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0.001", 2023-07-14T11:29:12.940 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "0.999", 2023-07-14T11:29:12.940 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.940 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "aggressiveness of optimization", 2023-07-14T11:29:12.940 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2023-07-14T11:29:12.940 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.941 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.941 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.941 INFO:teuthology.orchestra.run.smithi040.stdout: "end_time": { 2023-07-14T11:29:12.941 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_time", 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2359", 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.942 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.943 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.943 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ending time of day to automatically balance", 2023-07-14T11:29:12.943 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:12.943 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.943 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.944 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.944 INFO:teuthology.orchestra.run.smithi040.stdout: "end_weekday": { 2023-07-14T11:29:12.944 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_weekday", 2023-07-14T11:29:12.944 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:12.944 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.945 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.945 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:12.945 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:12.945 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:12.945 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.946 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2023-07-14T11:29:12.946 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:12.946 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.946 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.946 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.947 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.948 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.948 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.948 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.948 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.949 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.949 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.949 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.949 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.949 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.950 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.950 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.950 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.950 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.950 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.951 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.951 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:12.951 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:12.951 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.951 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.952 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.952 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.952 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.952 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.952 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.953 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.953 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.953 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.953 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.953 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.954 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:12.955 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.955 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.955 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.955 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:12.955 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:12.956 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:12.956 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:12.956 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:12.956 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:12.956 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.957 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.957 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.957 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.957 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.957 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.958 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.959 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.959 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.959 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.959 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: "min_score": { 2023-07-14T11:29:12.960 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_score", 2023-07-14T11:29:12.961 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.961 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.961 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.961 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:12.961 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.962 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.962 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.962 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "minimum score, below which no optimization is attempted", 2023-07-14T11:29:12.962 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.962 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.963 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.963 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.963 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:12.963 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:12.964 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.964 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.964 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.964 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "upmap", 2023-07-14T11:29:12.964 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.965 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.965 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:12.965 INFO:teuthology.orchestra.run.smithi040.stdout: "crush-compat", 2023-07-14T11:29:12.965 INFO:teuthology.orchestra.run.smithi040.stdout: "none", 2023-07-14T11:29:12.965 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap" 2023-07-14T11:29:12.966 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:12.966 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Balancer mode", 2023-07-14T11:29:12.966 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.966 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.967 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.967 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.967 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_ids": { 2023-07-14T11:29:12.967 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_ids", 2023-07-14T11:29:12.968 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.968 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.968 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.968 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:12.968 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.969 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.969 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.969 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "pools which the automatic balancing will be limited to", 2023-07-14T11:29:12.969 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.969 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.970 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.970 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.970 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:12.970 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:12.970 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.971 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.971 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.971 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:12.971 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.972 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.972 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.972 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and attempt optimization", 2023-07-14T11:29:12.972 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.972 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.973 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.973 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.973 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_deviation": { 2023-07-14T11:29:12.973 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_deviation", 2023-07-14T11:29:12.973 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.974 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.974 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.974 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:12.974 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:12.974 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "deviation below which no optimization is attempted", 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.975 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.976 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_optimizations": { 2023-07-14T11:29:12.976 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_optimizations", 2023-07-14T11:29:12.976 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:12.976 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.976 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:12.977 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:12.977 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.977 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.977 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.978 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum upmap optimizations to make per attempt", 2023-07-14T11:29:12.978 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.978 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.978 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.978 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.979 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:12.979 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.979 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:12.979 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cephadm", 2023-07-14T11:29:12.979 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:12.980 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:12.980 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:12.980 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_down_multiplier": { 2023-07-14T11:29:12.980 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_down_multiplier", 2023-07-14T11:29:12.980 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:12.981 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.981 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.981 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:12.981 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.981 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.982 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.983 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_refresh_rate": { 2023-07-14T11:29:12.983 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_refresh_rate", 2023-07-14T11:29:12.983 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.983 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.983 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.984 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "20", 2023-07-14T11:29:12.984 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.984 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.984 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.984 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2023-07-14T11:29:12.985 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.985 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.985 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.985 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.985 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_starting_port": { 2023-07-14T11:29:12.986 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_starting_port", 2023-07-14T11:29:12.986 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:12.986 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.986 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.986 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4721", 2023-07-14T11:29:12.987 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.987 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.987 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.987 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2023-07-14T11:29:12.987 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.988 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.988 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.988 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.988 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_password": { 2023-07-14T11:29:12.988 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_password", 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.989 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.990 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.990 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web password", 2023-07-14T11:29:12.990 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.990 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.990 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.991 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.991 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_user": { 2023-07-14T11:29:12.991 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_user", 2023-07-14T11:29:12.991 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:12.991 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.992 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.992 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:12.992 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.992 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.993 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.993 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web user", 2023-07-14T11:29:12.993 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.993 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.993 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.994 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.994 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_ptrace": { 2023-07-14T11:29:12.994 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_ptrace", 2023-07-14T11:29:12.994 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:12.995 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.995 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.995 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:12.995 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.995 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.996 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.996 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2023-07-14T11:29:12.996 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2023-07-14T11:29:12.996 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:12.997 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:12.997 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:12.997 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_interval": { 2023-07-14T11:29:12.997 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_interval", 2023-07-14T11:29:12.997 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:12.998 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:12.998 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:12.998 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:12.998 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:12.999 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:12.999 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:12.999 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to autotune daemon memory", 2023-07-14T11:29:12.999 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:12.999 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.000 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.000 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.000 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_memory_target_ratio": { 2023-07-14T11:29:13.000 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_memory_target_ratio", 2023-07-14T11:29:13.000 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:13.001 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.001 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.001 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.7", 2023-07-14T11:29:13.001 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.001 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.002 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.002 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2023-07-14T11:29:13.002 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.002 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.002 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.003 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.003 INFO:teuthology.orchestra.run.smithi040.stdout: "cgroups_split": { 2023-07-14T11:29:13.003 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cgroups_split", 2023-07-14T11:29:13.003 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.003 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.004 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.004 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.004 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.004 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.004 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.005 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2023-07-14T11:29:13.005 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.005 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.005 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.005 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.006 INFO:teuthology.orchestra.run.smithi040.stdout: "config_checks_enabled": { 2023-07-14T11:29:13.006 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_checks_enabled", 2023-07-14T11:29:13.006 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.006 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.006 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2023-07-14T11:29:13.007 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.008 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.008 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.008 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.008 INFO:teuthology.orchestra.run.smithi040.stdout: "config_dashboard": { 2023-07-14T11:29:13.008 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_dashboard", 2023-07-14T11:29:13.009 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.009 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.009 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.009 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.009 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.010 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.010 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.010 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2023-07-14T11:29:13.010 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.010 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.011 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.011 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.011 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_alertmanager": { 2023-07-14T11:29:13.011 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_alertmanager", 2023-07-14T11:29:13.011 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.012 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.012 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.012 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2023-07-14T11:29:13.012 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.013 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.013 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.013 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:13.013 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.014 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.014 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.014 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.014 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_base": { 2023-07-14T11:29:13.014 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_base", 2023-07-14T11:29:13.015 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.015 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.015 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.015 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph", 2023-07-14T11:29:13.015 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.016 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.016 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.016 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Container image name, without the tag", 2023-07-14T11:29:13.016 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.016 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.017 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.017 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.017 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_elasticsearch": { 2023-07-14T11:29:13.017 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_elasticsearch", 2023-07-14T11:29:13.017 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.018 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.018 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.018 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2023-07-14T11:29:13.018 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "elasticsearch container image", 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.019 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.020 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.020 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_grafana": { 2023-07-14T11:29:13.020 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_grafana", 2023-07-14T11:29:13.020 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.021 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.021 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.021 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2023-07-14T11:29:13.021 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.021 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.022 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.022 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:13.022 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.022 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.022 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.023 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.023 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_haproxy": { 2023-07-14T11:29:13.023 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_haproxy", 2023-07-14T11:29:13.023 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.023 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.024 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.024 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2023-07-14T11:29:13.024 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.024 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.024 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.025 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "HAproxy container image", 2023-07-14T11:29:13.025 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.025 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.025 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.026 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.026 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_agent": { 2023-07-14T11:29:13.026 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_agent", 2023-07-14T11:29:13.026 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.026 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.027 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.027 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2023-07-14T11:29:13.027 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.027 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.028 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.028 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger agent container image", 2023-07-14T11:29:13.028 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.028 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.028 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.029 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.029 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_collector": { 2023-07-14T11:29:13.029 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_collector", 2023-07-14T11:29:13.029 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.029 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.030 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.030 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2023-07-14T11:29:13.030 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.030 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.031 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.031 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger collector container image", 2023-07-14T11:29:13.031 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.031 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.032 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.032 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.032 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_query": { 2023-07-14T11:29:13.032 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_query", 2023-07-14T11:29:13.032 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.033 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.033 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.033 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2023-07-14T11:29:13.033 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.033 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.034 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.034 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger query container image", 2023-07-14T11:29:13.034 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.034 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.035 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.035 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.035 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_keepalived": { 2023-07-14T11:29:13.035 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_keepalived", 2023-07-14T11:29:13.035 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.036 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.036 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.036 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2023-07-14T11:29:13.036 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Keepalived container image", 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.037 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.038 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.038 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_loki": { 2023-07-14T11:29:13.038 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_loki", 2023-07-14T11:29:13.038 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.038 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.039 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.039 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2023-07-14T11:29:13.039 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.039 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.039 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.040 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Loki container image", 2023-07-14T11:29:13.040 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.040 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.040 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.040 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.041 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_node_exporter": { 2023-07-14T11:29:13.041 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_node_exporter", 2023-07-14T11:29:13.041 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.041 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.041 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.042 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2023-07-14T11:29:13.042 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.042 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.042 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.042 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:13.043 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.043 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.043 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.043 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.043 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_prometheus": { 2023-07-14T11:29:13.044 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_prometheus", 2023-07-14T11:29:13.044 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.044 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.044 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.044 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2023-07-14T11:29:13.045 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.045 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.045 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.045 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:13.045 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.046 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.046 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.046 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.046 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_promtail": { 2023-07-14T11:29:13.046 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_promtail", 2023-07-14T11:29:13.047 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.047 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.047 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.047 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2023-07-14T11:29:13.047 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.048 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.048 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.048 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Promtail container image", 2023-07-14T11:29:13.048 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.048 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.049 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.049 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.049 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_snmp_gateway": { 2023-07-14T11:29:13.049 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_snmp_gateway", 2023-07-14T11:29:13.049 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.050 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.050 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.050 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2023-07-14T11:29:13.050 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.050 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.051 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.051 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SNMP Gateway container image", 2023-07-14T11:29:13.051 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.051 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.051 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.052 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.052 INFO:teuthology.orchestra.run.smithi040.stdout: "container_init": { 2023-07-14T11:29:13.052 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_init", 2023-07-14T11:29:13.052 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.052 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.053 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.053 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.053 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.053 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.053 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.054 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Run podman/docker with `--init`", 2023-07-14T11:29:13.054 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.054 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.054 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.054 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.055 INFO:teuthology.orchestra.run.smithi040.stdout: "daemon_cache_timeout": { 2023-07-14T11:29:13.055 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "daemon_cache_timeout", 2023-07-14T11:29:13.055 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.055 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.055 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.056 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:13.056 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.056 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.056 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.056 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache service (daemon) inventory", 2023-07-14T11:29:13.057 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.057 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.057 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.057 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.057 INFO:teuthology.orchestra.run.smithi040.stdout: "default_cephadm_command_timeout": { 2023-07-14T11:29:13.058 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_cephadm_command_timeout", 2023-07-14T11:29:13.058 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.058 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.058 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.058 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "900", 2023-07-14T11:29:13.059 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.059 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.059 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.059 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2023-07-14T11:29:13.059 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.060 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.060 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.060 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.060 INFO:teuthology.orchestra.run.smithi040.stdout: "default_registry": { 2023-07-14T11:29:13.060 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_registry", 2023-07-14T11:29:13.061 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.061 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.061 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.061 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io", 2023-07-14T11:29:13.061 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.062 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.062 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.062 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2023-07-14T11:29:13.062 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.062 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: "device_cache_timeout": { 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_cache_timeout", 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.063 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.064 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.064 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1800", 2023-07-14T11:29:13.064 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.064 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.064 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.065 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache device inventory", 2023-07-14T11:29:13.065 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.065 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.065 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.065 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.066 INFO:teuthology.orchestra.run.smithi040.stdout: "device_enhanced_scan": { 2023-07-14T11:29:13.066 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_enhanced_scan", 2023-07-14T11:29:13.066 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.066 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.066 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.067 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.067 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.067 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.067 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.068 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use libstoragemgmt during device scans", 2023-07-14T11:29:13.068 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.068 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.068 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.068 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.069 INFO:teuthology.orchestra.run.smithi040.stdout: "facts_cache_timeout": { 2023-07-14T11:29:13.069 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "facts_cache_timeout", 2023-07-14T11:29:13.069 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.069 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.070 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.070 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:13.070 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.070 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.070 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.071 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache host facts data", 2023-07-14T11:29:13.071 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.071 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.071 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.071 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.072 INFO:teuthology.orchestra.run.smithi040.stdout: "host_check_interval": { 2023-07-14T11:29:13.072 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "host_check_interval", 2023-07-14T11:29:13.072 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.072 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.072 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.073 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:13.073 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.073 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.073 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.073 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to perform a host check", 2023-07-14T11:29:13.074 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.074 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.074 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.074 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.075 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.076 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.076 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.076 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.076 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.076 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.077 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.077 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.077 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.077 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.077 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.078 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.079 INFO:teuthology.orchestra.run.smithi040.stdout: "log_refresh_metadata": { 2023-07-14T11:29:13.079 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_refresh_metadata", 2023-07-14T11:29:13.079 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.079 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.079 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.080 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.080 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.080 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.080 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.080 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2023-07-14T11:29:13.081 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.081 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.081 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.081 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.081 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.082 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.082 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.082 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.082 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.083 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.083 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.083 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.083 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.083 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2023-07-14T11:29:13.084 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.084 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.084 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.084 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.084 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.085 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.085 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.085 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.085 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.085 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.086 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.086 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.086 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.086 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.087 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.087 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.087 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.087 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.087 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.088 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.088 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.088 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.088 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.088 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.089 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.089 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.089 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.089 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.089 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.090 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.091 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.091 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.091 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.091 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.091 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf": { 2023-07-14T11:29:13.092 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf", 2023-07-14T11:29:13.092 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.092 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.092 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.092 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.093 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.093 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.093 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.093 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2023-07-14T11:29:13.093 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.094 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.094 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.094 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.094 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2023-07-14T11:29:13.095 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2023-07-14T11:29:13.095 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.095 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.095 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.095 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "*", 2023-07-14T11:29:13.096 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.096 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.096 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.096 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2023-07-14T11:29:13.096 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.097 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.097 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.097 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.097 INFO:teuthology.orchestra.run.smithi040.stdout: "max_count_per_host": { 2023-07-14T11:29:13.097 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_count_per_host", 2023-07-14T11:29:13.098 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.098 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.098 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.098 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.099 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.099 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.099 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.099 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of daemons per service per host", 2023-07-14T11:29:13.099 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.100 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.100 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.100 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.100 INFO:teuthology.orchestra.run.smithi040.stdout: "max_osd_draining_count": { 2023-07-14T11:29:13.100 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_osd_draining_count", 2023-07-14T11:29:13.101 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.101 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.101 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.101 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.102 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.102 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.102 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.102 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2023-07-14T11:29:13.102 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: "migration_current": { 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "migration_current", 2023-07-14T11:29:13.103 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.104 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.104 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.104 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.104 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "internal - do not modify", 2023-07-14T11:29:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.106 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.106 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.106 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:13.106 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:13.106 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.107 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.107 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.107 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "root", 2023-07-14T11:29:13.107 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.107 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.108 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.108 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm-package", 2023-07-14T11:29:13.108 INFO:teuthology.orchestra.run.smithi040.stdout: "root" 2023-07-14T11:29:13.108 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.108 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "mode for remote execution of cephadm", 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_alerts_path": { 2023-07-14T11:29:13.109 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_alerts_path", 2023-07-14T11:29:13.110 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.110 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.110 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.110 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2023-07-14T11:29:13.110 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.111 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.111 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.111 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "location of alerts to include in prometheus deployments", 2023-07-14T11:29:13.111 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.111 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.112 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.112 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.112 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_password": { 2023-07-14T11:29:13.112 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_password", 2023-07-14T11:29:13.112 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.113 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.113 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.113 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:13.113 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.113 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.114 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.114 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web password", 2023-07-14T11:29:13.114 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.114 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.114 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.115 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.115 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_user": { 2023-07-14T11:29:13.115 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_user", 2023-07-14T11:29:13.115 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.115 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.116 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.116 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:13.116 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.116 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.116 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.117 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web user", 2023-07-14T11:29:13.117 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.117 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.117 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.117 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.118 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_insecure": { 2023-07-14T11:29:13.118 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_insecure", 2023-07-14T11:29:13.118 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.118 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.118 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.119 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.119 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.119 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.119 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.120 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2023-07-14T11:29:13.120 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.120 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.120 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.120 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.121 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_password": { 2023-07-14T11:29:13.121 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_password", 2023-07-14T11:29:13.121 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.121 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.121 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.122 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.122 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.122 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.122 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.122 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_url": { 2023-07-14T11:29:13.123 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_url", 2023-07-14T11:29:13.124 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.124 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.124 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.124 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.124 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.125 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.125 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.125 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2023-07-14T11:29:13.125 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.125 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.126 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.126 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.126 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_username": { 2023-07-14T11:29:13.126 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_username", 2023-07-14T11:29:13.126 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.127 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.127 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.127 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.127 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.127 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.128 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.128 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2023-07-14T11:29:13.128 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.128 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.129 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.129 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.129 INFO:teuthology.orchestra.run.smithi040.stdout: "secure_monitoring_stack": { 2023-07-14T11:29:13.129 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "secure_monitoring_stack", 2023-07-14T11:29:13.129 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.130 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.130 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.130 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.130 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.131 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.131 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.131 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2023-07-14T11:29:13.131 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.131 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.132 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.132 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.132 INFO:teuthology.orchestra.run.smithi040.stdout: "service_discovery_port": { 2023-07-14T11:29:13.132 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "service_discovery_port", 2023-07-14T11:29:13.132 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.133 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.133 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.133 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8765", 2023-07-14T11:29:13.133 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.133 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.134 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.134 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "cephadm service discovery port", 2023-07-14T11:29:13.134 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.134 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.134 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.135 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.135 INFO:teuthology.orchestra.run.smithi040.stdout: "ssh_config_file": { 2023-07-14T11:29:13.135 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssh_config_file", 2023-07-14T11:29:13.135 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.135 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.136 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.136 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.136 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.136 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.136 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.137 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "customized SSH config file to connect to managed hosts", 2023-07-14T11:29:13.137 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.137 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.137 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.137 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.138 INFO:teuthology.orchestra.run.smithi040.stdout: "use_agent": { 2023-07-14T11:29:13.138 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_agent", 2023-07-14T11:29:13.138 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.138 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.138 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.139 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.139 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.139 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.139 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.139 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2023-07-14T11:29:13.140 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.140 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.140 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.140 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.140 INFO:teuthology.orchestra.run.smithi040.stdout: "use_repo_digest": { 2023-07-14T11:29:13.141 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_repo_digest", 2023-07-14T11:29:13.141 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.141 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.141 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.141 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.142 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.142 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.142 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.142 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2023-07-14T11:29:13.142 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.143 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.143 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.143 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.143 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_failed_host_check": { 2023-07-14T11:29:13.143 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_failed_host_check", 2023-07-14T11:29:13.144 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.144 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.144 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.144 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.144 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.145 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.145 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.145 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if the host check fails", 2023-07-14T11:29:13.145 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.145 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.146 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.146 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.146 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_daemons": { 2023-07-14T11:29:13.146 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_daemons", 2023-07-14T11:29:13.146 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.147 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.147 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.147 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.147 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.147 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.148 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.148 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2023-07-14T11:29:13.148 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.148 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.148 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_hosts": { 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_hosts", 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.149 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.150 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.150 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.150 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.150 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.150 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2023-07-14T11:29:13.151 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.151 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.151 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.151 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.151 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.152 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.152 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.152 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crash", 2023-07-14T11:29:13.152 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.152 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.153 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.153 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.153 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.153 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.153 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.154 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.155 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.155 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.155 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.155 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.155 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.156 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.156 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.156 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.156 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.156 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.157 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.157 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.157 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.157 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.157 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.158 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.158 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.158 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.158 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.158 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.159 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.159 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.159 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.159 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.159 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.160 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.160 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.160 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.160 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.160 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.161 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.161 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.161 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.161 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.161 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.162 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.162 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.162 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.162 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.162 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.163 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.163 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.163 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.163 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.163 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.164 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.164 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.164 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.164 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.164 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.165 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.166 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.166 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.166 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.166 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.166 INFO:teuthology.orchestra.run.smithi040.stdout: "retain_interval": { 2023-07-14T11:29:13.167 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retain_interval", 2023-07-14T11:29:13.167 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.167 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.167 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.167 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "31536000", 2023-07-14T11:29:13.168 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.168 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.168 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.168 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain crashes before pruning them", 2023-07-14T11:29:13.168 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.169 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.169 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.169 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.169 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_recent_interval": { 2023-07-14T11:29:13.169 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_recent_interval", 2023-07-14T11:29:13.170 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.170 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.170 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.170 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1209600", 2023-07-14T11:29:13.170 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.171 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.171 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.171 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "time interval in which to warn about recent crashes", 2023-07-14T11:29:13.171 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.171 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.172 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.172 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.172 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.172 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.172 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.173 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dashboard", 2023-07-14T11:29:13.173 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.173 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.173 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.173 INFO:teuthology.orchestra.run.smithi040.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2023-07-14T11:29:13.174 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2023-07-14T11:29:13.174 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.174 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.174 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.174 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.175 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.175 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.175 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.175 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.175 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.176 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.176 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.176 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.176 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_HOST": { 2023-07-14T11:29:13.176 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_HOST", 2023-07-14T11:29:13.177 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.177 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.177 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.177 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.178 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.179 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.179 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.179 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2023-07-14T11:29:13.179 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2023-07-14T11:29:13.179 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.180 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.180 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.180 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.180 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.180 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.181 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.181 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.181 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.181 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.181 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_ENABLED": { 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_ENABLED", 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.182 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.183 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.183 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.183 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.183 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.183 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.184 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.184 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.184 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.184 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.184 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_LOG_PAYLOAD": { 2023-07-14T11:29:13.185 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2023-07-14T11:29:13.185 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.185 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.185 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.185 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.186 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.186 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.186 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.186 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.186 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: "ENABLE_BROWSABLE_API": { 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ENABLE_BROWSABLE_API", 2023-07-14T11:29:13.187 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.188 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.188 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.188 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.188 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.189 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.189 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.189 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.189 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.189 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.190 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.190 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.190 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_CEPHFS": { 2023-07-14T11:29:13.190 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2023-07-14T11:29:13.190 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.191 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.191 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.191 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.191 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.191 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.192 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.192 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.192 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.192 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.192 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.193 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.193 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2023-07-14T11:29:13.193 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2023-07-14T11:29:13.193 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.193 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.194 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.194 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.194 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.194 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.194 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.195 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.195 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.195 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.195 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.195 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.196 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_ISCSI": { 2023-07-14T11:29:13.196 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2023-07-14T11:29:13.196 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.196 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.196 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.197 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.197 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.197 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.197 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.197 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_MIRRORING": { 2023-07-14T11:29:13.198 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2023-07-14T11:29:13.199 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.199 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.199 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.199 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.200 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.200 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.200 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.200 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.200 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.201 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.201 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.201 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.201 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_NFS": { 2023-07-14T11:29:13.201 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_NFS", 2023-07-14T11:29:13.202 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.202 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.202 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.202 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.202 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.203 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.203 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.203 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.203 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.203 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.204 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.204 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.204 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RBD": { 2023-07-14T11:29:13.204 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RBD", 2023-07-14T11:29:13.204 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.205 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.205 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.205 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.205 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.205 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.206 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.206 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.206 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.206 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.206 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.207 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.207 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RGW": { 2023-07-14T11:29:13.207 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RGW", 2023-07-14T11:29:13.207 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.207 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.208 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.208 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.208 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.208 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.208 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.209 INFO:teuthology.orchestra.run.smithi040.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2023-07-14T11:29:13.210 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2023-07-14T11:29:13.210 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.210 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.210 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.211 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.212 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.212 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.212 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.212 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_PASSWORD": { 2023-07-14T11:29:13.212 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_PASSWORD", 2023-07-14T11:29:13.213 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.213 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.213 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.213 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:13.214 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.214 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.214 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.214 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.214 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_SSL_VERIFY": { 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2023-07-14T11:29:13.215 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.216 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.216 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.216 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.216 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.217 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.217 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.217 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.217 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.217 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_URL": { 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_URL", 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.218 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.219 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.219 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.219 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.219 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.219 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.220 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.220 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.220 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.220 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.220 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.221 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_USERNAME": { 2023-07-14T11:29:13.221 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_USERNAME", 2023-07-14T11:29:13.221 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.221 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.221 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.222 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:13.222 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.222 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.222 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.222 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.223 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.223 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.223 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.223 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.223 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_FRONTEND_API_URL": { 2023-07-14T11:29:13.224 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2023-07-14T11:29:13.224 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.224 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.224 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.224 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.225 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.225 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.225 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.225 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.225 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2023-07-14T11:29:13.226 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.227 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.227 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.227 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.227 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.227 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.228 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.228 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.228 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.228 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.228 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.229 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.229 INFO:teuthology.orchestra.run.smithi040.stdout: "ISCSI_API_SSL_VERIFICATION": { 2023-07-14T11:29:13.229 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2023-07-14T11:29:13.229 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.229 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.230 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.230 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.230 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.230 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.230 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.231 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.231 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.231 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.231 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.231 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.232 INFO:teuthology.orchestra.run.smithi040.stdout: "ISSUE_TRACKER_API_KEY": { 2023-07-14T11:29:13.232 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISSUE_TRACKER_API_KEY", 2023-07-14T11:29:13.232 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.232 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.232 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.233 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.233 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.233 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.233 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.233 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.234 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.234 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.234 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.234 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.234 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_HOST": { 2023-07-14T11:29:13.235 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_HOST", 2023-07-14T11:29:13.235 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.235 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.235 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.235 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.236 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.236 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.236 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.236 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.236 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.237 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.237 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.237 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.237 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2023-07-14T11:29:13.237 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.238 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.239 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.239 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.239 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.239 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.239 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.240 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.240 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2023-07-14T11:29:13.240 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2023-07-14T11:29:13.240 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.241 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.242 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.242 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.242 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.242 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.243 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.244 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.244 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.244 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.244 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.245 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.245 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.245 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.245 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.245 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.246 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2023-07-14T11:29:13.246 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2023-07-14T11:29:13.246 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.246 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.246 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.247 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.247 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.247 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.247 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.247 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.248 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.248 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.248 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.248 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.248 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2023-07-14T11:29:13.249 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2023-07-14T11:29:13.249 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.249 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.249 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.249 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.250 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.250 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.250 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.250 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.250 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.251 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.251 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.251 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.251 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2023-07-14T11:29:13.251 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2023-07-14T11:29:13.252 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.252 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.252 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.252 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.253 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.253 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.253 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.253 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.253 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.254 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.254 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.254 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.254 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2023-07-14T11:29:13.254 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2023-07-14T11:29:13.255 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.255 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.255 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.255 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.255 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.256 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.256 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.256 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.256 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.257 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.257 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.257 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.257 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2023-07-14T11:29:13.257 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2023-07-14T11:29:13.258 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.258 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.258 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.258 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.258 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.259 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.259 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.259 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.259 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.259 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.260 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.260 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.260 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_ENABLED": { 2023-07-14T11:29:13.260 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_ENABLED", 2023-07-14T11:29:13.260 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.261 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.261 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.261 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.261 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.261 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.262 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.263 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2023-07-14T11:29:13.263 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2023-07-14T11:29:13.263 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.263 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.263 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.264 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2023-07-14T11:29:13.264 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.264 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.264 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.264 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.265 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.265 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.265 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.265 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.265 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2023-07-14T11:29:13.266 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2023-07-14T11:29:13.266 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.266 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.266 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.266 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.267 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.267 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.267 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.267 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.267 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.268 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.268 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.268 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.268 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_LENGTH": { 2023-07-14T11:29:13.268 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2023-07-14T11:29:13.269 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.269 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.269 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.269 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8", 2023-07-14T11:29:13.269 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.270 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.270 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.270 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.270 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.270 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.271 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.271 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.271 INFO:teuthology.orchestra.run.smithi040.stdout: "REST_REQUESTS_TIMEOUT": { 2023-07-14T11:29:13.271 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "REST_REQUESTS_TIMEOUT", 2023-07-14T11:29:13.271 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.272 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.272 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.272 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "45", 2023-07-14T11:29:13.272 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.272 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.273 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.273 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.273 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.273 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.273 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.274 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.274 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ACCESS_KEY": { 2023-07-14T11:29:13.274 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ACCESS_KEY", 2023-07-14T11:29:13.274 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.274 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.275 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.275 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.275 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.275 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.275 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.276 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.276 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.276 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.276 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.277 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.277 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ADMIN_RESOURCE": { 2023-07-14T11:29:13.277 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2023-07-14T11:29:13.277 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.277 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.278 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.278 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:13.278 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.278 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.278 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.279 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.279 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.279 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.279 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.279 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.280 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SECRET_KEY": { 2023-07-14T11:29:13.280 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SECRET_KEY", 2023-07-14T11:29:13.280 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.280 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.280 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.281 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.281 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.281 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.281 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.281 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.282 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.282 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.282 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.282 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.282 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SSL_VERIFY": { 2023-07-14T11:29:13.283 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SSL_VERIFY", 2023-07-14T11:29:13.283 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.283 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.283 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.283 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.284 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.284 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.284 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.284 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.284 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.285 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.285 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.285 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.285 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_SPAN": { 2023-07-14T11:29:13.285 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2023-07-14T11:29:13.286 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.286 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.286 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.286 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:13.287 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.287 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.287 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.287 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.287 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.288 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.288 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.288 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.288 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2023-07-14T11:29:13.288 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2023-07-14T11:29:13.289 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.289 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.289 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.289 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.289 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.290 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.291 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.291 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2023-07-14T11:29:13.291 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2023-07-14T11:29:13.291 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.291 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.292 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.292 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:13.292 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.292 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.292 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.293 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.293 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.293 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.293 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.293 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.294 INFO:teuthology.orchestra.run.smithi040.stdout: "cross_origin_url": { 2023-07-14T11:29:13.294 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cross_origin_url", 2023-07-14T11:29:13.294 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.294 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.295 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.295 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.295 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.295 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.296 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.296 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.296 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.296 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: "crt_file": { 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crt_file", 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.297 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.298 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.298 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.298 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.298 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.299 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.299 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.299 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.299 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.299 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.300 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.300 INFO:teuthology.orchestra.run.smithi040.stdout: "debug": { 2023-07-14T11:29:13.300 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "debug", 2023-07-14T11:29:13.300 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.300 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.301 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.301 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.301 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.301 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.301 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.302 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable/disable debug options", 2023-07-14T11:29:13.302 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.302 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.302 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.302 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.303 INFO:teuthology.orchestra.run.smithi040.stdout: "jwt_token_ttl": { 2023-07-14T11:29:13.303 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "jwt_token_ttl", 2023-07-14T11:29:13.303 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.303 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.303 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.304 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "28800", 2023-07-14T11:29:13.304 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.304 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.304 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.304 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.305 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.305 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.305 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.305 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.305 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:13.306 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:13.306 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.306 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.306 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.306 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.307 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.307 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.307 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.307 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.307 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.308 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.308 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.308 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.309 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.309 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.309 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.309 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.309 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.310 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.311 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.311 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.311 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.311 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.312 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.312 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.312 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.312 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.312 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.313 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.313 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.313 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.313 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.313 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.314 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.314 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.314 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.314 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.314 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.315 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.315 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.315 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.315 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.316 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.316 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.316 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.316 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.316 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.317 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.317 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.317 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.317 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.317 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.318 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.319 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.319 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.319 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.319 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.319 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.320 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.320 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.320 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.320 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.320 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.321 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.321 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.321 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.321 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.322 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.322 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.322 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.322 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.322 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.323 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.323 INFO:teuthology.orchestra.run.smithi040.stdout: "motd": { 2023-07-14T11:29:13.323 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "motd", 2023-07-14T11:29:13.323 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.324 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.324 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.324 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.324 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.324 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.325 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.325 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "The message of the day", 2023-07-14T11:29:13.325 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.325 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.325 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.326 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.326 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect_resolve_ip_addr": { 2023-07-14T11:29:13.326 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "redirect_resolve_ip_addr", 2023-07-14T11:29:13.327 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.327 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.327 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.327 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.327 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.328 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.328 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.328 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.328 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.328 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.329 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.329 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.329 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:13.329 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:13.329 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.330 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.330 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.330 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:13.330 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.331 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.331 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.331 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.331 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.331 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.332 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.332 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.332 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:13.332 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:13.332 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.333 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.333 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.333 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8080", 2023-07-14T11:29:13.333 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.334 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.334 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.334 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.334 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.334 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.335 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.335 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.335 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:13.335 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:13.335 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.336 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.337 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.337 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.337 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.337 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl_server_port": { 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl_server_port", 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.338 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.339 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8443", 2023-07-14T11:29:13.339 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.339 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.339 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.339 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.340 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.340 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.340 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.340 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.340 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:13.341 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:13.341 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.341 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.341 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "redirect", 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.342 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect" 2023-07-14T11:29:13.343 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.343 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.343 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.343 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.343 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.344 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.344 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:13.344 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:13.344 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.344 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.345 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.345 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:13.345 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:13.345 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:13.345 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.346 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.346 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.346 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.346 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.346 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.347 INFO:teuthology.orchestra.run.smithi040.stdout: "url_prefix": { 2023-07-14T11:29:13.347 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url_prefix", 2023-07-14T11:29:13.347 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.347 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.347 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.348 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.349 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.349 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.349 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.349 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.349 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.350 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.350 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "devicehealth", 2023-07-14T11:29:13.350 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.350 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.350 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.351 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_monitoring": { 2023-07-14T11:29:13.351 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_monitoring", 2023-07-14T11:29:13.351 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.351 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.351 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "monitor device health metrics", 2023-07-14T11:29:13.352 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.353 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.353 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.353 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.353 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.353 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.354 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.354 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.354 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.354 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.354 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.355 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.355 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.355 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.355 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.355 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.356 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.356 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.356 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.356 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.356 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.357 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.357 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.357 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.357 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.357 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.358 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.358 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.358 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.358 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.359 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.360 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.360 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.360 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.360 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.361 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.361 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.361 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.361 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.361 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.362 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.362 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.362 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.362 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.362 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.363 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.363 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.363 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.363 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.363 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.364 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.364 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.364 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.364 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.364 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.365 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.365 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.365 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.366 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.367 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.367 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.367 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.367 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.367 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.368 INFO:teuthology.orchestra.run.smithi040.stdout: "mark_out_threshold": { 2023-07-14T11:29:13.368 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mark_out_threshold", 2023-07-14T11:29:13.368 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.368 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.369 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.369 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2419200", 2023-07-14T11:29:13.369 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.369 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.369 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.370 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically mark OSD if it may fail before this long", 2023-07-14T11:29:13.370 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.370 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.370 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.370 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.371 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_name": { 2023-07-14T11:29:13.371 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_name", 2023-07-14T11:29:13.371 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.371 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.371 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.372 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "device_health_metrics", 2023-07-14T11:29:13.372 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.372 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.372 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.372 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name of pool in which to store device health metrics", 2023-07-14T11:29:13.373 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.373 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.373 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.373 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.373 INFO:teuthology.orchestra.run.smithi040.stdout: "retention_period": { 2023-07-14T11:29:13.374 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retention_period", 2023-07-14T11:29:13.374 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.374 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.374 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.374 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15552000", 2023-07-14T11:29:13.375 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.375 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.375 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.375 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain device health metrics", 2023-07-14T11:29:13.375 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.376 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.376 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.376 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.376 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_frequency": { 2023-07-14T11:29:13.376 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_frequency", 2023-07-14T11:29:13.377 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.377 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.377 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.377 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:13.377 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.378 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.378 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.378 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to scrape device health metrics", 2023-07-14T11:29:13.378 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.378 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.379 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.379 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.379 INFO:teuthology.orchestra.run.smithi040.stdout: "self_heal": { 2023-07-14T11:29:13.379 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "self_heal", 2023-07-14T11:29:13.379 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.380 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.380 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.380 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.380 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.381 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.381 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.381 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "preemptively heal cluster around devices that may fail", 2023-07-14T11:29:13.381 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.381 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.382 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.383 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.383 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:13.383 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.383 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.383 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.384 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and check device health", 2023-07-14T11:29:13.384 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.384 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.384 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.384 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.385 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_threshold": { 2023-07-14T11:29:13.385 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_threshold", 2023-07-14T11:29:13.385 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.385 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.385 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.386 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7257600", 2023-07-14T11:29:13.386 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.386 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.386 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.386 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise health warning if OSD may fail before this long", 2023-07-14T11:29:13.387 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.387 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.387 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.387 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.387 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "diskprediction_local", 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.388 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.389 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.389 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.389 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.389 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.389 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.390 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.390 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.390 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.390 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.390 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.391 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.391 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.391 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.391 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.391 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.392 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.392 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.392 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.392 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.393 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.394 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.394 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.394 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.394 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.394 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.395 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.395 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.395 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.395 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.395 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.396 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.396 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.396 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.396 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.396 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.397 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.397 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.397 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.397 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.397 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.398 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.398 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.398 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.398 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.398 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.399 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.399 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.399 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.399 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.399 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.400 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.400 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.400 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.400 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.401 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.401 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.401 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.401 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.401 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.402 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.402 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.402 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.402 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.402 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.403 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.403 INFO:teuthology.orchestra.run.smithi040.stdout: "predict_interval": { 2023-07-14T11:29:13.403 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predict_interval", 2023-07-14T11:29:13.403 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.403 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.404 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.404 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:13.404 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.404 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.404 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.405 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.405 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.405 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.405 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.405 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.406 INFO:teuthology.orchestra.run.smithi040.stdout: "predictor_model": { 2023-07-14T11:29:13.406 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predictor_model", 2023-07-14T11:29:13.406 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.406 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.406 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.407 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "prophetstor", 2023-07-14T11:29:13.407 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.407 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.407 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.407 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.408 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.408 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.408 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.408 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.408 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:13.409 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:13.409 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.409 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.409 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.409 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.410 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.411 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.411 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.411 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.411 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.411 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.412 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "influx", 2023-07-14T11:29:13.412 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": false, 2023-07-14T11:29:13.412 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "influxdb python module not found", 2023-07-14T11:29:13.412 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.412 INFO:teuthology.orchestra.run.smithi040.stdout: "batch_size": { 2023-07-14T11:29:13.413 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "batch_size", 2023-07-14T11:29:13.413 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.413 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.413 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.413 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5000", 2023-07-14T11:29:13.414 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.414 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.414 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.414 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2023-07-14T11:29:13.414 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.415 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.415 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.415 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.415 INFO:teuthology.orchestra.run.smithi040.stdout: "database": { 2023-07-14T11:29:13.415 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "database", 2023-07-14T11:29:13.416 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.416 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.416 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.416 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "ceph", 2023-07-14T11:29:13.416 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.417 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.417 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.417 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2023-07-14T11:29:13.417 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.417 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.418 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.418 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.418 INFO:teuthology.orchestra.run.smithi040.stdout: "hostname": { 2023-07-14T11:29:13.418 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "hostname", 2023-07-14T11:29:13.418 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.419 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.419 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.419 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.419 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.419 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.420 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.420 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server hostname", 2023-07-14T11:29:13.420 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.420 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.421 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.421 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.421 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:13.421 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:13.421 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.422 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.422 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.422 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "30", 2023-07-14T11:29:13.422 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "5", 2023-07-14T11:29:13.422 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.423 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.424 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.424 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.424 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.424 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.424 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.425 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.425 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.425 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.425 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.425 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.426 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.426 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.426 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.426 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.426 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.427 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.427 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.427 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.427 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.427 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.428 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.428 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.428 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.428 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.428 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.429 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.429 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.429 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.429 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.429 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.430 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.431 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.431 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.431 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.431 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.431 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.432 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.432 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.432 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.432 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.432 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.433 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.433 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.433 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.433 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.433 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.434 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.434 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.434 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.434 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.434 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.435 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.436 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.436 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.436 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.436 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.436 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.437 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.437 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.437 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.437 INFO:teuthology.orchestra.run.smithi040.stdout: "password": { 2023-07-14T11:29:13.437 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "password", 2023-07-14T11:29:13.438 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.438 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.438 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.438 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.438 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.439 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.439 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.439 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "password of InfluxDB server user", 2023-07-14T11:29:13.439 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.439 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.440 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.440 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.440 INFO:teuthology.orchestra.run.smithi040.stdout: "port": { 2023-07-14T11:29:13.440 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "port", 2023-07-14T11:29:13.440 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.441 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.441 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.441 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8086", 2023-07-14T11:29:13.441 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.441 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.442 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.442 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server port", 2023-07-14T11:29:13.442 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.442 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.442 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.443 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.443 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:13.443 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:13.443 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.443 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.444 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.444 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "false", 2023-07-14T11:29:13.444 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.444 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.445 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.445 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:13.445 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.445 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.445 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.446 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.446 INFO:teuthology.orchestra.run.smithi040.stdout: "threads": { 2023-07-14T11:29:13.446 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threads", 2023-07-14T11:29:13.446 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.446 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.447 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.447 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:13.447 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:13.447 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "32", 2023-07-14T11:29:13.448 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.448 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2023-07-14T11:29:13.448 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.448 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.448 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.449 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.449 INFO:teuthology.orchestra.run.smithi040.stdout: "username": { 2023-07-14T11:29:13.449 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "username", 2023-07-14T11:29:13.449 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.449 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.450 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.450 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.450 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.450 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.450 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.451 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "username of InfluxDB server user", 2023-07-14T11:29:13.451 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.451 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.451 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.451 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.452 INFO:teuthology.orchestra.run.smithi040.stdout: "verify_ssl": { 2023-07-14T11:29:13.452 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "verify_ssl", 2023-07-14T11:29:13.452 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.452 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.452 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.453 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "true", 2023-07-14T11:29:13.453 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.453 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.453 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.453 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:13.454 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.454 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.454 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.454 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.454 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.455 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.455 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.455 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "insights", 2023-07-14T11:29:13.455 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.455 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.456 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.456 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.456 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.456 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.456 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.457 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.457 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.457 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.457 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.457 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.458 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.458 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.458 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.458 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.458 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.459 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.459 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.459 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.459 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.459 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.460 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.460 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.460 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.460 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.460 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.461 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.461 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.461 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.461 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.461 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.462 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.462 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.462 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.462 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.462 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.463 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.463 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.463 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.463 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.463 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.464 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.464 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.464 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.464 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.464 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.465 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.465 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.465 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.465 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.465 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.466 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.466 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.466 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.466 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.466 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.467 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.467 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.467 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.467 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.467 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.468 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.468 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.468 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.468 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.468 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.469 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.469 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.469 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.469 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.469 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.470 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.470 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.470 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.470 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.470 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "iostat", 2023-07-14T11:29:13.471 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.471 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.471 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.471 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.471 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.472 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.472 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.472 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.472 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.472 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.473 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.473 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.473 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.473 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.473 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.474 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.475 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.475 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.475 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.475 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.475 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.476 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.476 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.476 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.476 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.476 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.477 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.477 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.477 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.477 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.477 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.478 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.478 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.478 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.479 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.479 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.479 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.479 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.479 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.480 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.480 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.480 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.480 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.480 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.481 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.481 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.481 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.481 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.481 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.482 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.482 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.482 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.482 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.482 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.483 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.483 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.483 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.483 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.483 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.484 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.484 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.484 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.484 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.484 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.485 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.485 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.485 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.485 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.485 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "k8sevents", 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.486 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.487 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_event_retention_days": { 2023-07-14T11:29:13.487 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ceph_event_retention_days", 2023-07-14T11:29:13.487 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.487 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.487 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.488 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7", 2023-07-14T11:29:13.488 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.488 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.488 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.488 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Days to hold ceph event information within local cache", 2023-07-14T11:29:13.489 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.489 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.489 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.489 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.489 INFO:teuthology.orchestra.run.smithi040.stdout: "config_check_secs": { 2023-07-14T11:29:13.490 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_check_secs", 2023-07-14T11:29:13.490 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.490 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.490 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.490 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.491 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "10", 2023-07-14T11:29:13.491 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.491 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.491 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2023-07-14T11:29:13.491 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.492 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.492 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.492 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.492 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.492 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.493 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.493 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.493 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.493 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.493 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.494 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.494 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.494 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.494 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.494 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.495 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.496 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.496 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.496 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.496 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.496 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.497 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.497 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.497 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.497 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.497 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.498 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.498 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.498 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.498 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.498 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.499 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.499 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.499 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.499 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.499 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.500 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.500 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.500 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.500 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.500 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.501 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.501 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.501 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.501 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.501 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.502 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.502 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.502 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.502 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.502 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.503 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.503 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.503 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.503 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.503 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.504 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.504 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.504 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.504 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.504 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.505 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.505 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.505 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.505 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.505 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.506 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.506 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.506 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.506 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.506 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.507 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "localpool", 2023-07-14T11:29:13.507 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.507 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.507 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.507 INFO:teuthology.orchestra.run.smithi040.stdout: "failure_domain": { 2023-07-14T11:29:13.508 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "failure_domain", 2023-07-14T11:29:13.508 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.508 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.508 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.508 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "host", 2023-07-14T11:29:13.509 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.509 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.509 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.509 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "failure domain for any created local pool", 2023-07-14T11:29:13.509 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.510 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.511 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.511 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.511 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.511 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.511 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.512 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.512 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.512 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.512 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.512 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.513 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.513 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.513 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.513 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.513 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.514 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.514 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.514 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.514 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.514 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.515 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.515 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.515 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.515 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.515 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.516 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.516 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.516 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.516 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.516 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.517 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.517 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.517 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.517 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.517 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.518 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.518 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.518 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.518 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.518 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.519 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.519 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.519 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.519 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.519 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.520 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.520 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.520 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.520 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.520 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.521 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.521 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.521 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.521 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.521 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.522 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.522 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.522 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.522 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.522 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.523 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.524 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.524 INFO:teuthology.orchestra.run.smithi040.stdout: "min_size": { 2023-07-14T11:29:13.524 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_size", 2023-07-14T11:29:13.524 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.524 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.525 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.525 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.525 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.525 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.525 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.526 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default min_size for any created local pool", 2023-07-14T11:29:13.526 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2023-07-14T11:29:13.526 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.526 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.526 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.527 INFO:teuthology.orchestra.run.smithi040.stdout: "num_rep": { 2023-07-14T11:29:13.527 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "num_rep", 2023-07-14T11:29:13.527 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.527 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.527 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.528 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3", 2023-07-14T11:29:13.528 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.528 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.528 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.528 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default replica count for any created local pool", 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_num": { 2023-07-14T11:29:13.529 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_num", 2023-07-14T11:29:13.530 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.530 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.530 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.530 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "128", 2023-07-14T11:29:13.530 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.531 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.531 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.531 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default pg_num for any created local pool", 2023-07-14T11:29:13.531 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.531 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: "prefix": { 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prefix", 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.532 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.533 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.533 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.533 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.533 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.533 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.534 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name prefix for any created local pool", 2023-07-14T11:29:13.534 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.534 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.534 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.534 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "subtree": { 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "subtree", 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.535 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "rack", 2023-07-14T11:29:13.536 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.536 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.536 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.536 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "CRUSH level for which to create a local pool", 2023-07-14T11:29:13.537 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2023-07-14T11:29:13.537 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.537 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.537 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.537 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.538 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.538 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.538 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mds_autoscaler", 2023-07-14T11:29:13.538 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.538 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.539 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.539 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.539 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.539 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.539 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.540 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.540 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.540 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.540 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.541 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.541 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.541 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.541 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.541 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.542 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.542 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.542 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.542 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.542 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.543 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.543 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.543 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.543 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.543 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.544 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.544 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.544 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.544 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.544 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.545 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.546 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.546 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.546 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.546 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.546 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.547 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.547 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.547 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.547 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.547 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.548 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.548 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.548 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.548 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.548 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.549 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.550 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.550 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.550 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.550 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.550 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.551 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.551 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.551 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.551 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.551 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.552 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.552 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.552 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.553 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.553 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.553 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.553 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.554 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.554 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirroring", 2023-07-14T11:29:13.554 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.554 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.554 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.555 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.555 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.555 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.555 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.555 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.556 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.556 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.556 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.556 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.556 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.557 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.557 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.557 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.557 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.557 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.558 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.558 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.558 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.558 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.558 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.559 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.559 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.559 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.559 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.559 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.560 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.560 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.560 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.560 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.560 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.561 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.561 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.561 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.561 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.561 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.562 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.562 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.562 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.562 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.562 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.563 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.563 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.563 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.563 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.564 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.564 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.564 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.564 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.565 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.565 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.565 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.565 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.565 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.566 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.566 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.566 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.566 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.566 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.567 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.567 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.567 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.567 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.568 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.568 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.568 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.568 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.569 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.569 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.569 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.569 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.569 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.570 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.570 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.570 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "nfs", 2023-07-14T11:29:13.570 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.570 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.571 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.571 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.571 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.571 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.571 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.572 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.572 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.572 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.572 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.572 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.573 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.573 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.573 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.573 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.573 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.574 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.574 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.574 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.574 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.574 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.575 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.575 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.575 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.575 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.575 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.576 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.576 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.576 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.576 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.576 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.577 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.577 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.577 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.577 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.577 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.578 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.578 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.578 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.578 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.578 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.579 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.579 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.579 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.579 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.579 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.580 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.581 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.581 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.581 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.581 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.581 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.582 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.582 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.582 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.582 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.582 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.583 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.583 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.583 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.583 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.583 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.584 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.584 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.584 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.584 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.584 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.585 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.585 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.585 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.585 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:13.585 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.586 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.586 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.586 INFO:teuthology.orchestra.run.smithi040.stdout: "fail_fs": { 2023-07-14T11:29:13.586 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "fail_fs", 2023-07-14T11:29:13.586 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.587 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.587 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.587 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.587 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.587 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.588 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.588 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2023-07-14T11:29:13.588 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.588 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.588 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.589 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.590 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.590 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.590 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.590 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.590 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.591 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.591 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.591 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.591 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.591 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.592 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.592 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.592 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.592 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.592 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.593 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.593 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.593 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.593 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.593 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.594 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.594 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.594 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.594 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.594 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.595 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.595 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.595 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.595 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.595 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.596 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.596 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.596 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.596 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.596 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.597 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.598 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.598 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.598 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.598 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.598 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.599 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.599 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.599 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.599 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.599 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.600 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.600 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.600 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.600 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.600 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.601 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.601 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.601 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.601 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.601 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.602 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.602 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.602 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.602 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.602 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator": { 2023-07-14T11:29:13.603 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:13.603 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.603 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.603 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.603 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.604 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.604 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.604 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.604 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm", 2023-07-14T11:29:13.604 INFO:teuthology.orchestra.run.smithi040.stdout: "rook", 2023-07-14T11:29:13.605 INFO:teuthology.orchestra.run.smithi040.stdout: "test_orchestrator" 2023-07-14T11:29:13.605 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.605 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Orchestrator backend", 2023-07-14T11:29:13.605 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.605 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.606 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.606 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.606 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.606 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.606 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.607 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_perf_query", 2023-07-14T11:29:13.607 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.607 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.607 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.608 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.609 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.609 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.609 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.609 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.609 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.610 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.610 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.610 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.610 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.610 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.611 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.611 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.611 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.611 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.611 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.612 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.612 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.612 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.612 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.612 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.613 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.613 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.613 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.613 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.613 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.614 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.615 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.615 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.615 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.615 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.615 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.616 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.616 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.616 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.616 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.617 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.618 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.618 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.618 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.618 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.618 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.619 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.619 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.619 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.619 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.619 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.620 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.620 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.620 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.620 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.620 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.621 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.621 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.621 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.621 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.621 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.622 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.622 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_support", 2023-07-14T11:29:13.622 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.622 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.622 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.623 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.623 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.623 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.623 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.623 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.624 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.624 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.624 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.624 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.624 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.625 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.625 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.625 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.625 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.625 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.626 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.626 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.626 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.626 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.626 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.627 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.627 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.627 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.627 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.627 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.628 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.628 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.628 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.628 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.628 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.629 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.629 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.629 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.629 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.629 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.630 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.630 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.630 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.630 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.630 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.631 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.631 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.631 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.631 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.632 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.632 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.632 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.632 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.632 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.633 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.633 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.633 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.633 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.633 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.634 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.634 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.634 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.634 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.634 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.635 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.635 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.635 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.635 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.635 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.636 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.636 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.636 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.636 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.636 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.637 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.637 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.637 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.637 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.637 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_autoscaler", 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.638 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.639 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.639 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.639 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.639 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.639 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.640 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.640 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.640 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.640 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.640 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.641 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.641 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.641 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.641 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.641 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.642 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.642 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.642 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.642 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.642 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.643 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.644 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.644 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.644 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.644 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.644 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.645 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.645 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.645 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.645 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.645 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.646 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.646 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.646 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.646 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.646 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.647 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.647 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.647 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.647 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.647 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.648 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.648 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.648 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.648 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.648 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.649 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.649 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.649 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.649 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.649 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.650 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.650 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.650 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.650 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.650 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.651 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.651 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.651 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.651 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.651 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.652 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.652 INFO:teuthology.orchestra.run.smithi040.stdout: "noautoscale": { 2023-07-14T11:29:13.652 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "noautoscale", 2023-07-14T11:29:13.652 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.652 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.653 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.653 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.653 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.653 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.653 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.654 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "global autoscale flag", 2023-07-14T11:29:13.654 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Option to turn on/off the autoscaler for all pools", 2023-07-14T11:29:13.654 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.654 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.654 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.655 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:13.656 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.656 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.656 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.656 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.657 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.657 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.657 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.657 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.657 INFO:teuthology.orchestra.run.smithi040.stdout: "threshold": { 2023-07-14T11:29:13.658 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threshold", 2023-07-14T11:29:13.658 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:13.658 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.658 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.658 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:13.659 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1.0", 2023-07-14T11:29:13.659 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.659 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.659 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "scaling threshold", 2023-07-14T11:29:13.659 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2023-07-14T11:29:13.660 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.660 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.660 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.660 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.660 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.661 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.661 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "progress", 2023-07-14T11:29:13.661 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.661 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.661 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_pg_recovery_event": { 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_pg_recovery_event", 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.662 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.663 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.663 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.663 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.663 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow the module to show pg recovery progress", 2023-07-14T11:29:13.663 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.664 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.664 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.664 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.664 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:13.664 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:13.665 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.665 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.665 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.665 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.665 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.666 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.667 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.667 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.667 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.667 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.667 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.668 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.668 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.668 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.668 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.668 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.669 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.669 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.669 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.669 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.669 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.670 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.670 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.670 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.670 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.670 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.671 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.671 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.671 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.671 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.671 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.672 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.672 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.672 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.672 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.673 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.674 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.674 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.674 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.674 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.674 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.675 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.675 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.675 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.675 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.675 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.676 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.676 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.676 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.676 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.676 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.677 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.677 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.677 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.677 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.677 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.678 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.678 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.678 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.678 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.678 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.679 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.679 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.679 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.679 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.679 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.680 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.680 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.680 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.680 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.681 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.681 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.681 INFO:teuthology.orchestra.run.smithi040.stdout: "max_completed_events": { 2023-07-14T11:29:13.681 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_completed_events", 2023-07-14T11:29:13.681 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.682 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.682 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.682 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "50", 2023-07-14T11:29:13.682 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.682 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.683 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.683 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "number of past completed events to remember", 2023-07-14T11:29:13.683 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.683 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.683 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.684 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.684 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:13.684 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:13.684 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.684 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.685 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.685 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:13.685 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.685 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.685 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.686 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long the module is going to sleep", 2023-07-14T11:29:13.686 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.686 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.686 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.686 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.687 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.687 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.687 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.687 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus", 2023-07-14T11:29:13.687 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.688 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.688 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.688 INFO:teuthology.orchestra.run.smithi040.stdout: "cache": { 2023-07-14T11:29:13.688 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cache", 2023-07-14T11:29:13.688 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.689 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.689 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.689 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.689 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.689 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.690 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.690 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.690 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.690 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: "exclude_perf_counters": { 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "exclude_perf_counters", 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.691 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.692 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.692 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.692 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.692 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.693 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.693 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Do not include perf-counters in the metrics output", 2023-07-14T11:29:13.693 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2023-07-14T11:29:13.693 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.693 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.694 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.694 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.694 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.694 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.694 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.695 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.695 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.695 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.695 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.695 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.696 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.696 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.696 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.696 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.696 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.697 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.697 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.697 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.697 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.697 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.698 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.698 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.698 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.698 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.698 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.699 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.699 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.699 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.699 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.699 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.700 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.700 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.700 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.700 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.701 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.701 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.701 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.701 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.701 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.702 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.702 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.702 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.702 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.702 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.703 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.703 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.703 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.703 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.703 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.704 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.704 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.704 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.704 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.704 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.705 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.705 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.705 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.705 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.705 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.706 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.706 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.706 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.706 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.706 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.707 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.707 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.707 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.707 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.707 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.708 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.708 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.708 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools": { 2023-07-14T11:29:13.708 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools", 2023-07-14T11:29:13.708 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.709 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.709 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.709 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.709 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.709 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.710 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.710 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.710 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.710 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.710 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools_refresh_interval": { 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools_refresh_interval", 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.711 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.712 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300", 2023-07-14T11:29:13.712 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.712 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.712 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.712 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.713 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.713 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.713 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.713 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.714 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_interval": { 2023-07-14T11:29:13.714 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_interval", 2023-07-14T11:29:13.714 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:13.714 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.714 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.715 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15.0", 2023-07-14T11:29:13.715 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.715 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.715 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.715 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.716 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.716 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.716 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.716 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.716 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:13.717 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:13.717 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.717 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.717 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.717 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:13.718 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.718 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.718 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.718 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2023-07-14T11:29:13.718 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:13.719 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.720 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.720 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.720 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "9283", 2023-07-14T11:29:13.720 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.720 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the port on which the module listens for HTTP requests", 2023-07-14T11:29:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.721 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.722 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.722 INFO:teuthology.orchestra.run.smithi040.stdout: "stale_cache_strategy": { 2023-07-14T11:29:13.722 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stale_cache_strategy", 2023-07-14T11:29:13.722 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.722 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.723 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.723 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "log", 2023-07-14T11:29:13.723 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.723 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.723 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.724 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.724 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.724 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.724 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.724 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.725 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:13.725 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:13.725 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.725 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.725 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.726 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "default", 2023-07-14T11:29:13.726 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.726 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.726 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.726 INFO:teuthology.orchestra.run.smithi040.stdout: "default", 2023-07-14T11:29:13.727 INFO:teuthology.orchestra.run.smithi040.stdout: "error" 2023-07-14T11:29:13.727 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.727 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.727 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.727 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.728 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.728 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.728 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:13.728 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:13.728 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:13.729 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.730 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.730 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.730 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.730 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.730 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.731 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.731 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.731 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.731 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_support", 2023-07-14T11:29:13.731 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.732 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.732 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.732 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.732 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.732 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.733 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.733 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.733 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.733 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.733 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.734 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.734 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.734 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.734 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.734 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.735 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.735 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.735 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.735 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.735 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.736 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.736 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.736 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.736 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.736 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.737 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.737 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.737 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.737 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.737 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.738 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.738 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.738 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.738 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.738 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.739 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.739 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.739 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.739 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.739 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.740 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.740 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.740 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.740 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.740 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.741 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.741 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.741 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.741 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.741 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.742 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.742 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.742 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.742 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.742 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.743 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.743 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.743 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.743 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.743 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.744 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.744 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.744 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.744 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.745 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.745 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.745 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.745 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.746 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.746 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.746 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.746 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.746 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_snap_create": { 2023-07-14T11:29:13.747 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_snap_create", 2023-07-14T11:29:13.747 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.747 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.747 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.748 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:13.748 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.748 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.748 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.748 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.749 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.749 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.749 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.749 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.749 INFO:teuthology.orchestra.run.smithi040.stdout: "mirror_snapshot_schedule": { 2023-07-14T11:29:13.750 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirror_snapshot_schedule", 2023-07-14T11:29:13.750 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.750 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.750 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.751 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.751 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.751 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.751 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.751 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.752 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.752 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.752 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.752 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.753 INFO:teuthology.orchestra.run.smithi040.stdout: "trash_purge_schedule": { 2023-07-14T11:29:13.753 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "trash_purge_schedule", 2023-07-14T11:29:13.753 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.753 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.753 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.754 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.754 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.754 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.754 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.754 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.755 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.755 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.755 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.755 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.755 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.756 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.756 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.756 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "restful", 2023-07-14T11:29:13.756 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.756 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.757 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.757 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_auth": { 2023-07-14T11:29:13.757 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_auth", 2023-07-14T11:29:13.757 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.757 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.758 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.758 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.758 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.758 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.758 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.759 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.759 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.759 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.759 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.759 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.760 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:13.760 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:13.760 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.760 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.760 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.761 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.761 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.761 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.761 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.761 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.762 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.762 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.762 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.762 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.762 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.763 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.763 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.763 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.763 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.763 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.764 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.764 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.764 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.764 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.764 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.765 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.765 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.765 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.765 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.765 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.766 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.766 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.766 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.766 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.766 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.767 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.767 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.767 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.767 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.767 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.768 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.768 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.768 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.768 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.768 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.769 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.769 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.769 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.769 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.769 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.770 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.770 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.770 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.770 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.770 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.771 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.771 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.771 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.771 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.771 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.772 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.772 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.772 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.772 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.772 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.773 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.773 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.773 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.773 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.773 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.774 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.774 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.774 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.774 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.774 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.775 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.775 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.775 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.775 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.775 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.776 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.776 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.776 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.776 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.776 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:13.777 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:13.777 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.777 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.777 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.777 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.778 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.779 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.779 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.779 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:13.779 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:13.779 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.780 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.780 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.780 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.780 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.780 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.781 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.781 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.781 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.781 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.781 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.782 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.782 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.782 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.782 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.782 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rgw", 2023-07-14T11:29:13.783 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.783 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.783 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.783 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.783 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.784 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.784 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.784 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.784 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.784 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.785 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.785 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.785 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.785 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.785 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.786 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.786 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.786 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.786 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.786 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.787 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.787 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.787 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.787 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.787 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.788 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.788 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.788 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.788 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.789 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.790 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.790 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.790 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.790 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.790 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.791 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.791 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.791 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.791 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.791 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.792 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.792 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.792 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.792 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.792 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.793 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.793 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.793 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.793 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.793 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.794 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.794 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.794 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.794 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.794 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.795 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.795 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.795 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.795 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.795 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.796 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.796 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.796 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.796 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.796 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.797 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.797 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.797 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.797 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.797 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rook", 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.798 INFO:teuthology.orchestra.run.smithi040.stdout: "drive_group_interval": { 2023-07-14T11:29:13.799 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "drive_group_interval", 2023-07-14T11:29:13.799 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:13.799 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.799 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.799 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300.0", 2023-07-14T11:29:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval in seconds between re-application of applied drive_groups", 2023-07-14T11:29:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.801 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.801 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.801 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.801 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.801 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.802 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.802 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.802 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.802 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.802 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.803 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.803 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.803 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.803 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.803 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.804 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.804 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.804 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.804 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.804 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.805 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.806 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.806 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.806 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.806 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.806 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.807 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.807 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.807 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.807 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.807 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.808 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.808 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.808 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.808 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.808 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.809 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.809 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.809 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.809 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.809 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.810 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.810 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.810 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.810 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.810 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.811 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.811 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.811 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.811 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.811 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.812 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.812 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.812 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.812 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.812 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.813 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.813 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.813 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.813 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.813 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.814 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.814 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.814 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.814 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.814 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.815 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.815 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.815 INFO:teuthology.orchestra.run.smithi040.stdout: "storage_class": { 2023-07-14T11:29:13.815 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "storage_class", 2023-07-14T11:29:13.815 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.816 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.816 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.816 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "local", 2023-07-14T11:29:13.816 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.816 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.817 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.817 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "storage class name for LSO-discovered PVs", 2023-07-14T11:29:13.817 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.817 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.817 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.818 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.818 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.818 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.818 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.818 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "selftest", 2023-07-14T11:29:13.819 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.819 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.819 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.819 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.819 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.820 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.820 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.820 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.820 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.820 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.821 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.821 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.821 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.821 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.821 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.822 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.822 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.822 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.822 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.822 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.823 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.823 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.823 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.823 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.823 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.824 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.824 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.824 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.824 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.824 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.825 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.825 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.825 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.825 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.825 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.826 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.827 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.827 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.827 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.827 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.827 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.828 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.828 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.828 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.828 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.828 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.829 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.829 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.829 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.829 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.829 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.830 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.830 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.830 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.830 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.830 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.831 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.831 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.831 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.831 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.831 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.833 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.833 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.833 INFO:teuthology.orchestra.run.smithi040.stdout: "roption1": { 2023-07-14T11:29:13.833 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption1", 2023-07-14T11:29:13.833 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.834 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.834 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.834 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.834 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.834 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.835 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.836 INFO:teuthology.orchestra.run.smithi040.stdout: "roption2": { 2023-07-14T11:29:13.836 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption2", 2023-07-14T11:29:13.836 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.836 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.836 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.837 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "xyz", 2023-07-14T11:29:13.837 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.837 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.837 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.837 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.838 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.838 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.838 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.838 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.838 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption1": { 2023-07-14T11:29:13.839 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption1", 2023-07-14T11:29:13.839 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.839 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.839 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.839 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.840 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.840 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.840 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.840 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.840 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.841 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.841 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.841 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.841 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption2": { 2023-07-14T11:29:13.841 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption2", 2023-07-14T11:29:13.842 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.842 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.842 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.842 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.842 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.843 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.843 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.843 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.843 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.844 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.844 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.844 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.844 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption3": { 2023-07-14T11:29:13.844 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption3", 2023-07-14T11:29:13.845 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:13.845 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.845 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.845 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.845 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.846 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.846 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.846 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.846 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption4": { 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption4", 2023-07-14T11:29:13.847 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.848 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.848 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.848 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.848 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.848 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.849 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.849 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.849 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.849 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.849 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.850 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.850 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption5": { 2023-07-14T11:29:13.850 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption5", 2023-07-14T11:29:13.850 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.850 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.851 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.852 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.852 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.852 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.852 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.852 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption6": { 2023-07-14T11:29:13.853 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption6", 2023-07-14T11:29:13.853 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.853 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.853 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.853 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.854 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.854 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.854 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.854 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.854 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption7": { 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption7", 2023-07-14T11:29:13.855 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.856 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.856 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.856 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.856 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:13.856 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "42", 2023-07-14T11:29:13.857 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.857 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.857 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.857 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.857 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.858 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.858 INFO:teuthology.orchestra.run.smithi040.stdout: "testkey": { 2023-07-14T11:29:13.858 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testkey", 2023-07-14T11:29:13.858 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.858 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.859 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.859 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.859 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.859 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.859 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.860 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.860 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.860 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.860 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.860 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.861 INFO:teuthology.orchestra.run.smithi040.stdout: "testlkey": { 2023-07-14T11:29:13.861 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testlkey", 2023-07-14T11:29:13.861 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.861 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.861 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.862 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.862 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.862 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.862 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.862 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.863 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.863 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.863 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.863 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.863 INFO:teuthology.orchestra.run.smithi040.stdout: "testnewline": { 2023-07-14T11:29:13.864 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testnewline", 2023-07-14T11:29:13.864 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.864 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.864 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.864 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.865 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.865 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.865 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.865 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.865 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.866 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.866 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.866 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.866 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.866 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.867 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.867 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snap_schedule", 2023-07-14T11:29:13.867 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.867 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.867 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.868 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_m_granularity": { 2023-07-14T11:29:13.868 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_m_granularity", 2023-07-14T11:29:13.868 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.868 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.868 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.869 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.869 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.869 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.869 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.869 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow minute scheduled snapshots", 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: "dump_on_update": { 2023-07-14T11:29:13.870 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dump_on_update", 2023-07-14T11:29:13.871 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.871 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.871 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.871 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.871 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.872 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.872 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.872 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "dump database to debug log on update", 2023-07-14T11:29:13.872 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.872 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.873 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.873 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.873 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.873 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.873 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.874 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.874 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.874 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.874 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.874 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.875 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.876 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.876 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.876 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.876 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.877 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.878 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.878 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.878 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.878 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.878 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.879 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.879 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.879 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.879 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.879 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.880 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.880 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.880 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.880 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.880 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.881 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.881 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.881 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.881 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.881 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.882 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.882 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.882 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.882 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.882 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.883 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.883 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.883 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.883 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.883 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.884 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.885 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.885 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.885 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.885 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.886 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.886 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.886 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.886 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.886 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stats", 2023-07-14T11:29:13.887 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.888 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.888 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.888 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.888 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.888 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.889 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.889 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.889 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.889 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.889 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.890 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.890 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.890 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.890 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.890 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.891 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.891 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.891 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.891 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.891 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.892 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.892 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.892 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.892 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.892 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.893 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.894 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.894 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.894 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.894 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.894 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.895 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.895 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.895 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.895 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.895 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.896 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.896 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.896 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.896 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.896 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.897 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.897 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.897 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.897 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.897 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.898 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.898 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.898 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.898 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.898 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.899 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.899 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.899 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.899 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.899 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.901 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.901 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.901 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.901 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.901 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.902 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.902 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.902 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.902 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.902 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "status", 2023-07-14T11:29:13.903 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.903 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.903 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.903 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.903 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.904 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.904 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.904 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.904 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.904 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.905 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.905 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.905 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.905 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.905 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.906 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.906 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.906 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.906 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.906 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.907 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.907 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.907 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.907 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.907 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.908 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.908 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.908 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.908 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.908 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.909 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.909 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.909 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.909 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.909 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.910 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.910 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.910 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.910 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.910 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.911 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.911 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.911 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.911 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.911 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.912 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.912 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.912 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.912 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.912 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.913 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.913 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.913 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.913 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.913 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.914 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.914 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.914 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.914 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.914 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.915 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.916 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.916 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.916 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.916 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.916 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.917 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.917 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.917 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.917 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.917 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.918 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telegraf", 2023-07-14T11:29:13.918 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.918 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.918 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.918 INFO:teuthology.orchestra.run.smithi040.stdout: "address": { 2023-07-14T11:29:13.919 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "address", 2023-07-14T11:29:13.919 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.919 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.919 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.919 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2023-07-14T11:29:13.920 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.920 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.920 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.920 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.920 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.921 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.921 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.921 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.921 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:13.921 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:13.922 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:13.922 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.922 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.922 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15", 2023-07-14T11:29:13.922 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.923 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.923 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.923 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.923 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.923 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.924 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.924 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.924 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.924 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.924 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.925 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.925 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.925 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.925 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.925 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.926 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.926 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.926 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.926 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.926 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.927 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.927 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.927 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.927 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.927 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.928 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.928 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.928 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.928 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.928 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.929 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.929 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.929 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.929 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.929 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.930 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.930 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.930 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.930 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.930 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.931 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.932 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.932 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.932 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.932 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.932 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.933 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.933 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.933 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.933 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.933 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.934 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.934 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.934 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.934 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.934 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.935 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.935 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.935 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.935 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.935 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.936 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.936 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.936 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.936 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.936 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.937 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.937 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.937 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.937 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.937 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.938 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.938 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.938 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.938 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.938 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telemetry", 2023-07-14T11:29:13.939 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.939 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.939 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.939 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_basic": { 2023-07-14T11:29:13.939 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_basic", 2023-07-14T11:29:13.940 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.940 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.940 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.940 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.940 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.941 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.941 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.941 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share basic cluster information (size, version)", 2023-07-14T11:29:13.941 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.941 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.942 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.942 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.942 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_crash": { 2023-07-14T11:29:13.942 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_crash", 2023-07-14T11:29:13.942 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.943 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.944 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2023-07-14T11:29:13.944 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.944 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.944 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.944 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.945 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_device": { 2023-07-14T11:29:13.945 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_device", 2023-07-14T11:29:13.945 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.945 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.945 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.946 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:13.946 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.946 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.946 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.946 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2023-07-14T11:29:13.947 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.947 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.947 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.947 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.947 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_ident": { 2023-07-14T11:29:13.948 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_ident", 2023-07-14T11:29:13.948 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.948 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.948 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.948 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.949 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.949 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.949 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.949 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2023-07-14T11:29:13.949 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.950 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.950 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.950 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.950 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_perf": { 2023-07-14T11:29:13.950 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_perf", 2023-07-14T11:29:13.951 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.951 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.951 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.951 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.951 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.952 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.952 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.952 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share various performance metrics of a cluster", 2023-07-14T11:29:13.952 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.952 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.953 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.953 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.953 INFO:teuthology.orchestra.run.smithi040.stdout: "contact": { 2023-07-14T11:29:13.953 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "contact", 2023-07-14T11:29:13.953 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.954 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.954 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.954 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.954 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.954 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.955 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.955 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.955 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.955 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.955 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.956 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.956 INFO:teuthology.orchestra.run.smithi040.stdout: "description": { 2023-07-14T11:29:13.956 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "description", 2023-07-14T11:29:13.956 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.956 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.957 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.957 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.957 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.957 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.957 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.958 INFO:teuthology.orchestra.run.smithi040.stdout: "device_url": { 2023-07-14T11:29:13.959 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_url", 2023-07-14T11:29:13.959 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.959 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.959 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.959 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/device", 2023-07-14T11:29:13.960 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.960 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.960 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.960 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.960 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.961 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.961 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.961 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.961 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:13.961 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.962 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.963 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.963 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.963 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.963 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.963 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.964 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.964 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:13.964 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:13.964 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.964 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.965 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.965 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "24", 2023-07-14T11:29:13.965 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "8", 2023-07-14T11:29:13.965 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.965 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.966 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.966 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.966 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.966 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.966 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "last_opt_revision": { 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "last_opt_revision", 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.967 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1", 2023-07-14T11:29:13.968 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.968 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.968 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.968 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.968 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.969 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.969 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.969 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.969 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard": { 2023-07-14T11:29:13.969 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard", 2023-07-14T11:29:13.970 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.970 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.970 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.970 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.970 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.971 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.971 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.971 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.971 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.971 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.972 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.972 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.972 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard_description": { 2023-07-14T11:29:13.972 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard_description", 2023-07-14T11:29:13.972 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.973 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.973 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.973 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.973 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.973 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.974 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.974 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.974 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.974 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.974 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.975 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.975 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.975 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.975 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.975 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.976 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.976 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.976 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.976 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.976 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.977 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.977 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.977 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.977 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.977 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.978 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.978 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.978 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.978 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.978 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.979 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.979 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.979 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:13.979 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:13.979 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.980 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.980 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.980 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.980 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.980 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.981 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.981 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.981 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.981 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.981 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.982 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.982 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:13.982 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:13.982 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.982 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.983 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.983 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:13.983 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.983 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.983 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:13.984 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:13.984 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:13.984 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:13.984 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:13.984 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:13.985 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:13.985 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:13.985 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.985 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.985 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.986 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.986 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.986 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:13.986 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:13.986 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:13.987 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.987 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.987 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:13.987 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.987 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.988 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.988 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.988 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.988 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.988 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.989 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.989 INFO:teuthology.orchestra.run.smithi040.stdout: "organization": { 2023-07-14T11:29:13.989 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "organization", 2023-07-14T11:29:13.989 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.989 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.990 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.991 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.991 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.991 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.991 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.991 INFO:teuthology.orchestra.run.smithi040.stdout: "proxy": { 2023-07-14T11:29:13.992 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "proxy", 2023-07-14T11:29:13.992 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.992 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.992 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.992 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:13.993 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.993 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.993 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.993 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.993 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.994 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.994 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.994 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.994 INFO:teuthology.orchestra.run.smithi040.stdout: "url": { 2023-07-14T11:29:13.994 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url", 2023-07-14T11:29:13.995 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.995 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.995 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:13.995 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/report", 2023-07-14T11:29:13.995 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:13.996 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:13.996 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:13.996 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:13.996 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:13.996 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:13.997 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "test_orchestrator", 2023-07-14T11:29:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:13.998 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:13.999 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.000 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.000 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.000 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.000 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.000 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.001 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.001 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.001 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.001 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.001 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.002 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.002 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.002 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.002 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.002 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.003 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.003 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.003 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.003 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.003 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.004 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.004 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.004 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.004 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.004 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.005 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.006 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.006 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.006 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.006 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.006 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.007 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.007 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.007 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.007 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.007 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.008 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.008 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.008 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.008 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.008 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.009 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.009 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.009 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.009 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.009 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.010 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.011 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.011 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.011 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.011 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.011 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.012 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.012 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.012 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.012 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.013 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.013 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "volumes", 2023-07-14T11:29:14.013 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.013 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.013 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.014 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.014 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.014 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.014 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.014 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.015 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.015 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.015 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.015 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.015 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.016 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.016 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.016 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.016 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.016 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.017 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.017 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.017 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.017 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.017 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.018 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.018 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.018 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.018 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.018 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.019 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.019 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.019 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.019 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.019 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.020 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.020 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.020 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.020 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.020 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.021 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.021 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.021 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.021 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.021 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.022 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.022 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.022 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.022 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.022 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.023 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.024 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.024 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.024 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.024 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.024 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.025 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.025 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.025 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.025 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.025 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.026 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.026 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.026 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.026 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.026 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.027 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.027 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.027 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.027 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.027 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_clones": { 2023-07-14T11:29:14.028 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_clones", 2023-07-14T11:29:14.028 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.028 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.028 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4", 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Number of asynchronous cloner threads", 2023-07-14T11:29:14.029 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.030 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.030 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.030 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.030 INFO:teuthology.orchestra.run.smithi040.stdout: "snapshot_clone_delay": { 2023-07-14T11:29:14.031 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snapshot_clone_delay", 2023-07-14T11:29:14.031 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.031 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.031 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.031 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:14.032 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.032 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.032 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.032 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2023-07-14T11:29:14.032 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.033 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.033 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.033 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.033 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.033 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.034 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.034 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix", 2023-07-14T11:29:14.034 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.034 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.034 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.035 INFO:teuthology.orchestra.run.smithi040.stdout: "discovery_interval": { 2023-07-14T11:29:14.035 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "discovery_interval", 2023-07-14T11:29:14.035 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:14.035 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.035 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.036 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "100", 2023-07-14T11:29:14.036 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.036 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.036 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.036 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: "identifier": { 2023-07-14T11:29:14.037 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "identifier", 2023-07-14T11:29:14.038 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.038 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.038 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.038 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.038 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.039 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.039 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.039 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.039 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.039 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.040 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.040 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.040 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:14.040 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:14.040 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.041 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.041 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.041 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:14.041 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.041 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.042 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.042 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.042 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.042 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.042 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.043 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.043 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.043 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.043 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.043 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.044 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.044 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.044 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.044 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.044 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.045 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.045 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.045 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.045 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.045 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.046 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.046 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.046 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.046 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.046 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.047 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.047 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.047 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.047 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.047 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.048 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.048 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.048 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.048 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.048 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.049 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.050 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.050 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.050 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.050 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.050 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.051 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.051 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.051 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.051 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.051 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.052 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.052 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.052 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.052 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.052 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.053 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.053 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.053 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.053 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.053 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.054 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.054 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.054 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.054 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.054 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.055 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.055 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.055 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.055 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.055 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.056 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.056 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.056 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.056 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.056 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.057 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_host": { 2023-07-14T11:29:14.057 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_host", 2023-07-14T11:29:14.057 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.057 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.057 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.058 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.058 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.058 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.058 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.058 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.059 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.059 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.059 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.059 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.059 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_port": { 2023-07-14T11:29:14.060 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_port", 2023-07-14T11:29:14.060 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.060 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.060 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.060 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10051", 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.061 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.062 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.062 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.062 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_sender": { 2023-07-14T11:29:14.062 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_sender", 2023-07-14T11:29:14.062 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.063 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.063 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.063 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/usr/bin/zabbix_sender", 2023-07-14T11:29:14.063 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.063 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.064 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.064 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.064 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.064 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.064 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.065 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.066 INFO:teuthology.orchestra.run.smithi040.stdout: "modules": [ 2023-07-14T11:29:14.066 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm", 2023-07-14T11:29:14.066 INFO:teuthology.orchestra.run.smithi040.stdout: "dashboard", 2023-07-14T11:29:14.066 INFO:teuthology.orchestra.run.smithi040.stdout: "iostat", 2023-07-14T11:29:14.066 INFO:teuthology.orchestra.run.smithi040.stdout: "nfs", 2023-07-14T11:29:14.067 INFO:teuthology.orchestra.run.smithi040.stdout: "restful", 2023-07-14T11:29:14.067 INFO:teuthology.orchestra.run.smithi040.stdout: "snap_schedule" 2023-07-14T11:29:14.067 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.067 INFO:teuthology.orchestra.run.smithi040.stdout: "available_modules": [ 2023-07-14T11:29:14.067 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.068 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alerts", 2023-07-14T11:29:14.068 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.068 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.068 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.068 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:14.069 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:14.069 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.069 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.069 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.069 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How frequently to reexamine health status", 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.070 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.071 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.071 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.071 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.071 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.071 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.072 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.072 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.072 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.072 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.073 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.073 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.073 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.073 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.073 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.074 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.074 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.074 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.074 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.074 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.075 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.076 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.076 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.076 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.076 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.076 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.077 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.077 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.077 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.077 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.077 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.078 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.078 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.078 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.078 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.079 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.079 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.079 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.079 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.080 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.080 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.080 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.080 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.081 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.081 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.081 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.081 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.082 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.082 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.082 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.082 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.083 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.084 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.084 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.084 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.084 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.084 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.085 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.085 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.085 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.085 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.085 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.086 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.086 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_destination": { 2023-07-14T11:29:14.086 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_destination", 2023-07-14T11:29:14.086 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.086 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.087 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.087 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.087 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.087 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.087 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.088 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email address to send alerts to", 2023-07-14T11:29:14.088 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.088 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.088 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.088 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.089 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_from_name": { 2023-07-14T11:29:14.089 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_from_name", 2023-07-14T11:29:14.089 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.089 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.089 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.090 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "Ceph", 2023-07-14T11:29:14.090 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.090 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.090 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.090 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Email From: name", 2023-07-14T11:29:14.091 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.091 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.091 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.091 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.091 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_host": { 2023-07-14T11:29:14.092 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_host", 2023-07-14T11:29:14.092 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.092 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.092 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.092 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP server", 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.093 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.094 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.094 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.094 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_password": { 2023-07-14T11:29:14.094 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_password", 2023-07-14T11:29:14.094 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.095 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.095 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.095 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.095 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.095 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.096 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.096 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Password to authenticate with", 2023-07-14T11:29:14.096 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.096 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.096 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.097 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.097 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_port": { 2023-07-14T11:29:14.097 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_port", 2023-07-14T11:29:14.097 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.097 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.098 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.098 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "465", 2023-07-14T11:29:14.098 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.098 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.098 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.099 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP port", 2023-07-14T11:29:14.099 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.099 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.099 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.099 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.100 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_sender": { 2023-07-14T11:29:14.100 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_sender", 2023-07-14T11:29:14.100 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.100 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.100 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SMTP envelope sender", 2023-07-14T11:29:14.101 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.102 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.102 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.102 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.102 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_ssl": { 2023-07-14T11:29:14.102 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_ssl", 2023-07-14T11:29:14.103 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.103 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.103 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.103 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.103 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.104 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.104 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.104 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use SSL to connect to SMTP server", 2023-07-14T11:29:14.104 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.104 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.105 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.105 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.105 INFO:teuthology.orchestra.run.smithi040.stdout: "smtp_user": { 2023-07-14T11:29:14.105 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "smtp_user", 2023-07-14T11:29:14.105 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.106 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.106 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.106 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.106 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.106 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "User to authenticate as", 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.107 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.108 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.108 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.108 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.108 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "balancer", 2023-07-14T11:29:14.108 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.109 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.109 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.109 INFO:teuthology.orchestra.run.smithi040.stdout: "active": { 2023-07-14T11:29:14.109 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "active", 2023-07-14T11:29:14.109 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.110 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.110 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.110 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.110 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.110 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.111 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.111 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically balance PGs across cluster", 2023-07-14T11:29:14.111 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.111 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.111 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.112 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.112 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_time": { 2023-07-14T11:29:14.112 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_time", 2023-07-14T11:29:14.112 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.112 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.113 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.113 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0000", 2023-07-14T11:29:14.113 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.113 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.113 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.114 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "beginning time of day to automatically balance", 2023-07-14T11:29:14.114 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:14.114 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.114 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.114 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.115 INFO:teuthology.orchestra.run.smithi040.stdout: "begin_weekday": { 2023-07-14T11:29:14.115 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "begin_weekday", 2023-07-14T11:29:14.115 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:14.115 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.115 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.116 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:14.116 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:14.116 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:14.116 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.116 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to this day of the week or later", 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_max_iterations": { 2023-07-14T11:29:14.117 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_max_iterations", 2023-07-14T11:29:14.118 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:14.118 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.118 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.118 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "25", 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "250", 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum number of iterations to attempt optimization", 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.119 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.120 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.120 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.120 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_metrics": { 2023-07-14T11:29:14.120 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_metrics", 2023-07-14T11:29:14.120 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.121 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.121 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.121 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "pgs,objects,bytes", 2023-07-14T11:29:14.121 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.121 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.122 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.122 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "metrics with which to calculate OSD utilization", 2023-07-14T11:29:14.122 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.", 2023-07-14T11:29:14.122 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.122 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.123 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.123 INFO:teuthology.orchestra.run.smithi040.stdout: "crush_compat_step": { 2023-07-14T11:29:14.123 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crush_compat_step", 2023-07-14T11:29:14.123 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.123 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.124 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.124 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.5", 2023-07-14T11:29:14.124 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0.001", 2023-07-14T11:29:14.124 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "0.999", 2023-07-14T11:29:14.124 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.125 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "aggressiveness of optimization", 2023-07-14T11:29:14.125 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": ".99 is very aggressive, .01 is less aggressive", 2023-07-14T11:29:14.125 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.125 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.126 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.126 INFO:teuthology.orchestra.run.smithi040.stdout: "end_time": { 2023-07-14T11:29:14.126 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_time", 2023-07-14T11:29:14.126 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.126 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.127 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.127 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2359", 2023-07-14T11:29:14.127 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.127 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.127 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.128 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ending time of day to automatically balance", 2023-07-14T11:29:14.128 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "This is a time of day in the format HHMM.", 2023-07-14T11:29:14.128 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.128 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.129 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.129 INFO:teuthology.orchestra.run.smithi040.stdout: "end_weekday": { 2023-07-14T11:29:14.129 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "end_weekday", 2023-07-14T11:29:14.129 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:14.129 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "0", 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "6", 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.130 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Restrict automatic balancing to days of the week earlier than this", 2023-07-14T11:29:14.131 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "0 = Sunday, 1 = Monday, etc.", 2023-07-14T11:29:14.131 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.131 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.131 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.131 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.132 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.132 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.132 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.132 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.132 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.133 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.133 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.133 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.133 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.133 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.134 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.134 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.134 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.134 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.134 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.135 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.135 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.135 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.135 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.135 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.136 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.137 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.137 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.137 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.137 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.137 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.138 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.138 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.138 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.138 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.138 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.139 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.139 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.139 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.139 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.140 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.140 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.140 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.140 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.140 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.141 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.141 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.141 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.141 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.141 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.142 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.142 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.142 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.142 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.142 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.143 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.143 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.143 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.143 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.143 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.144 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.144 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.144 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.144 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.144 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: "min_score": { 2023-07-14T11:29:14.145 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_score", 2023-07-14T11:29:14.146 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.146 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.146 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.146 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:14.146 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.147 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.147 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.147 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "minimum score, below which no optimization is attempted", 2023-07-14T11:29:14.147 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.147 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.148 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.148 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.148 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:14.148 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:14.148 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.149 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.149 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.149 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "upmap", 2023-07-14T11:29:14.149 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.149 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.150 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.150 INFO:teuthology.orchestra.run.smithi040.stdout: "crush-compat", 2023-07-14T11:29:14.150 INFO:teuthology.orchestra.run.smithi040.stdout: "none", 2023-07-14T11:29:14.150 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap" 2023-07-14T11:29:14.150 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.151 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Balancer mode", 2023-07-14T11:29:14.151 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.151 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.151 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.151 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_ids": { 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_ids", 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.152 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.153 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.153 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.153 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.153 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "pools which the automatic balancing will be limited to", 2023-07-14T11:29:14.153 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.154 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.154 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.154 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.154 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:14.154 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:14.155 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.155 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.155 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.155 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:14.155 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.156 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.156 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.156 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and attempt optimization", 2023-07-14T11:29:14.156 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.156 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.157 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.157 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.157 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_deviation": { 2023-07-14T11:29:14.157 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_deviation", 2023-07-14T11:29:14.157 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.158 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.159 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "deviation below which no optimization is attempted", 2023-07-14T11:29:14.159 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "If the number of PGs are within this count then no optimization is attempted", 2023-07-14T11:29:14.159 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.159 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.159 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.160 INFO:teuthology.orchestra.run.smithi040.stdout: "upmap_max_optimizations": { 2023-07-14T11:29:14.160 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "upmap_max_optimizations", 2023-07-14T11:29:14.160 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:14.160 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.161 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "maximum upmap optimizations to make per attempt", 2023-07-14T11:29:14.162 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.162 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.162 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.162 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.163 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.163 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.163 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.163 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cephadm", 2023-07-14T11:29:14.163 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.164 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.164 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.164 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_down_multiplier": { 2023-07-14T11:29:14.164 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_down_multiplier", 2023-07-14T11:29:14.164 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.165 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.166 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Multiplied by agent refresh rate to calculate how long agent must not report before being marked down", 2023-07-14T11:29:14.166 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.166 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.166 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.166 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.167 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_refresh_rate": { 2023-07-14T11:29:14.167 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_refresh_rate", 2023-07-14T11:29:14.167 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.167 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.167 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.168 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "20", 2023-07-14T11:29:14.168 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.168 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.168 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.168 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How often agent on each host will try to gather and send metadata", 2023-07-14T11:29:14.169 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.169 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.169 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.169 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.169 INFO:teuthology.orchestra.run.smithi040.stdout: "agent_starting_port": { 2023-07-14T11:29:14.170 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "agent_starting_port", 2023-07-14T11:29:14.170 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.170 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.170 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.170 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4721", 2023-07-14T11:29:14.171 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.171 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.171 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.171 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)", 2023-07-14T11:29:14.172 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.172 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.172 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.172 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.172 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_password": { 2023-07-14T11:29:14.173 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_password", 2023-07-14T11:29:14.173 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.173 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.173 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.173 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web password", 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.174 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.175 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.175 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.175 INFO:teuthology.orchestra.run.smithi040.stdout: "alertmanager_web_user": { 2023-07-14T11:29:14.175 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "alertmanager_web_user", 2023-07-14T11:29:14.175 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.176 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.176 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.176 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.176 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.176 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.177 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.177 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Alertmanager web user", 2023-07-14T11:29:14.177 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.177 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.177 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.178 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.178 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_ptrace": { 2023-07-14T11:29:14.178 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_ptrace", 2023-07-14T11:29:14.178 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.178 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.179 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow SYS_PTRACE capability on ceph containers", 2023-07-14T11:29:14.180 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.", 2023-07-14T11:29:14.180 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.180 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.180 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.180 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_interval": { 2023-07-14T11:29:14.181 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_interval", 2023-07-14T11:29:14.181 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.181 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.181 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to autotune daemon memory", 2023-07-14T11:29:14.182 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.183 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.183 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.183 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.183 INFO:teuthology.orchestra.run.smithi040.stdout: "autotune_memory_target_ratio": { 2023-07-14T11:29:14.183 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "autotune_memory_target_ratio", 2023-07-14T11:29:14.184 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.184 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.184 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.184 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0.7", 2023-07-14T11:29:14.184 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.185 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.185 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.185 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "ratio of total system memory to divide amongst autotuned daemons", 2023-07-14T11:29:14.185 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.185 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: "cgroups_split": { 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cgroups_split", 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.186 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.187 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.187 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.187 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.187 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.187 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.188 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Pass --cgroups=split when cephadm creates containers (currently podman only)", 2023-07-14T11:29:14.188 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.188 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.188 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.188 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.189 INFO:teuthology.orchestra.run.smithi040.stdout: "config_checks_enabled": { 2023-07-14T11:29:14.189 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_checks_enabled", 2023-07-14T11:29:14.189 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.189 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.190 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.190 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.190 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.190 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.190 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.191 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable or disable the cephadm configuration analysis", 2023-07-14T11:29:14.191 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.191 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.191 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.191 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.192 INFO:teuthology.orchestra.run.smithi040.stdout: "config_dashboard": { 2023-07-14T11:29:14.192 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_dashboard", 2023-07-14T11:29:14.192 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.192 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.192 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.193 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.193 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.193 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.193 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.193 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "manage configs like API endpoints in Dashboard.", 2023-07-14T11:29:14.194 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.194 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.194 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.194 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.194 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_alertmanager": { 2023-07-14T11:29:14.195 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_alertmanager", 2023-07-14T11:29:14.195 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.195 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.195 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.195 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/alertmanager:v0.25.0", 2023-07-14T11:29:14.196 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.196 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.196 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.196 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:14.196 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.197 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.197 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.197 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.197 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_base": { 2023-07-14T11:29:14.197 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_base", 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph", 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.198 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.199 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.199 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Container image name, without the tag", 2023-07-14T11:29:14.199 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.199 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.199 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.200 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.200 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_elasticsearch": { 2023-07-14T11:29:14.200 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_elasticsearch", 2023-07-14T11:29:14.200 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.201 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.201 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.201 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/omrizeneva/elasticsearch:6.8.23", 2023-07-14T11:29:14.201 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.201 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.202 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.202 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "elasticsearch container image", 2023-07-14T11:29:14.202 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.202 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.202 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_grafana": { 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_grafana", 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.203 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.204 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/ceph-grafana:9.4.7", 2023-07-14T11:29:14.204 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.204 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.204 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.204 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:14.205 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.205 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.205 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.205 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.205 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_haproxy": { 2023-07-14T11:29:14.206 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_haproxy", 2023-07-14T11:29:14.206 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.206 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.206 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.206 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/haproxy:2.3", 2023-07-14T11:29:14.207 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.207 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.207 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.207 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "HAproxy container image", 2023-07-14T11:29:14.207 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.208 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.208 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.208 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.208 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_agent": { 2023-07-14T11:29:14.208 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_agent", 2023-07-14T11:29:14.209 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.209 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.209 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.209 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-agent:1.29", 2023-07-14T11:29:14.210 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.210 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.210 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.210 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger agent container image", 2023-07-14T11:29:14.210 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.211 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.211 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.211 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.211 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_collector": { 2023-07-14T11:29:14.211 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_collector", 2023-07-14T11:29:14.212 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.212 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.212 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.212 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-collector:1.29", 2023-07-14T11:29:14.213 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.213 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.213 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.213 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger collector container image", 2023-07-14T11:29:14.213 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.214 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.214 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.214 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.214 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_jaeger_query": { 2023-07-14T11:29:14.214 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_jaeger_query", 2023-07-14T11:29:14.215 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.215 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.215 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.215 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/jaegertracing/jaeger-query:1.29", 2023-07-14T11:29:14.215 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.216 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.216 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.216 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Jaeger query container image", 2023-07-14T11:29:14.216 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.216 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.217 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.217 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.217 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_keepalived": { 2023-07-14T11:29:14.217 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_keepalived", 2023-07-14T11:29:14.217 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.218 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.218 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.218 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/ceph/keepalived:2.2.4", 2023-07-14T11:29:14.218 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.218 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.219 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.219 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Keepalived container image", 2023-07-14T11:29:14.219 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.219 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.219 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.220 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.220 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_loki": { 2023-07-14T11:29:14.220 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_loki", 2023-07-14T11:29:14.220 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.220 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.221 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.221 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/loki:2.4.0", 2023-07-14T11:29:14.221 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.221 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.221 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.222 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Loki container image", 2023-07-14T11:29:14.222 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.222 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.222 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.223 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.223 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_node_exporter": { 2023-07-14T11:29:14.223 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_node_exporter", 2023-07-14T11:29:14.223 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.223 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.224 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.224 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/node-exporter:v1.5.0", 2023-07-14T11:29:14.224 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.224 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.224 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.225 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_prometheus": { 2023-07-14T11:29:14.226 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_prometheus", 2023-07-14T11:29:14.226 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.226 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.226 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.226 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "quay.io/prometheus/prometheus:v2.43.0", 2023-07-14T11:29:14.227 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.227 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.227 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.227 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus container image", 2023-07-14T11:29:14.227 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.228 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.228 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.228 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.228 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_promtail": { 2023-07-14T11:29:14.228 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_promtail", 2023-07-14T11:29:14.229 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.229 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.229 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.229 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/grafana/promtail:2.4.0", 2023-07-14T11:29:14.229 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.230 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.230 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.230 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Promtail container image", 2023-07-14T11:29:14.230 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.230 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: "container_image_snmp_gateway": { 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_image_snmp_gateway", 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.231 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.232 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.232 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io/maxwo/snmp-notifier:v1.2.1", 2023-07-14T11:29:14.232 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.232 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.232 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.233 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "SNMP Gateway container image", 2023-07-14T11:29:14.233 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.233 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.233 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.233 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.234 INFO:teuthology.orchestra.run.smithi040.stdout: "container_init": { 2023-07-14T11:29:14.234 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "container_init", 2023-07-14T11:29:14.234 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.234 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.234 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.235 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.235 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.235 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.235 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.235 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Run podman/docker with `--init`", 2023-07-14T11:29:14.236 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.236 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.236 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.236 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.236 INFO:teuthology.orchestra.run.smithi040.stdout: "daemon_cache_timeout": { 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "daemon_cache_timeout", 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:14.237 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.238 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.238 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.238 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache service (daemon) inventory", 2023-07-14T11:29:14.238 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.238 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.239 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.239 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.239 INFO:teuthology.orchestra.run.smithi040.stdout: "default_cephadm_command_timeout": { 2023-07-14T11:29:14.239 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_cephadm_command_timeout", 2023-07-14T11:29:14.239 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.240 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.240 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.240 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "900", 2023-07-14T11:29:14.240 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.240 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.241 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.241 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Default timeout applied to cephadm commands run directly on the host (in seconds)", 2023-07-14T11:29:14.241 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.241 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.241 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.242 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.242 INFO:teuthology.orchestra.run.smithi040.stdout: "default_registry": { 2023-07-14T11:29:14.242 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "default_registry", 2023-07-14T11:29:14.242 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.243 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.243 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.243 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "docker.io", 2023-07-14T11:29:14.243 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.243 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.244 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.244 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Search-registry to which we should normalize unqualified image names. This is not the default registry", 2023-07-14T11:29:14.244 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.244 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.244 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.245 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.245 INFO:teuthology.orchestra.run.smithi040.stdout: "device_cache_timeout": { 2023-07-14T11:29:14.245 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_cache_timeout", 2023-07-14T11:29:14.245 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.245 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.246 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.246 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1800", 2023-07-14T11:29:14.246 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.246 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.246 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.247 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache device inventory", 2023-07-14T11:29:14.247 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.247 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.247 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.247 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.248 INFO:teuthology.orchestra.run.smithi040.stdout: "device_enhanced_scan": { 2023-07-14T11:29:14.248 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_enhanced_scan", 2023-07-14T11:29:14.248 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.248 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.248 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.249 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.249 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.249 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.249 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.249 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use libstoragemgmt during device scans", 2023-07-14T11:29:14.250 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.250 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.250 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.250 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.250 INFO:teuthology.orchestra.run.smithi040.stdout: "facts_cache_timeout": { 2023-07-14T11:29:14.251 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "facts_cache_timeout", 2023-07-14T11:29:14.251 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.251 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.251 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.251 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:14.252 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.252 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.252 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.252 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "seconds to cache host facts data", 2023-07-14T11:29:14.252 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: "host_check_interval": { 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "host_check_interval", 2023-07-14T11:29:14.253 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.254 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.254 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.254 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:14.254 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.254 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.255 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.255 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to perform a host check", 2023-07-14T11:29:14.255 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.255 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.255 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.256 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.256 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.256 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.256 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.256 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.257 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.258 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.258 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.258 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.258 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.258 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.259 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.259 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.259 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.259 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.259 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.260 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.260 INFO:teuthology.orchestra.run.smithi040.stdout: "log_refresh_metadata": { 2023-07-14T11:29:14.260 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_refresh_metadata", 2023-07-14T11:29:14.260 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.260 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.261 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.261 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.261 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.261 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.261 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level", 2023-07-14T11:29:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.262 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.263 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.264 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.264 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.264 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.264 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.264 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "log to the \"cephadm\" cluster log channel\"", 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.265 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.266 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.266 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.266 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.266 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.266 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.267 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.267 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.267 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.267 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.267 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.268 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.268 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.268 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.268 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.268 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.269 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.269 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.269 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.269 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.269 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.270 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.270 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.270 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.270 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.270 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.271 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.272 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.272 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.272 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf": { 2023-07-14T11:29:14.272 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf", 2023-07-14T11:29:14.272 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.273 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.273 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.273 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.273 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.273 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.274 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.274 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Manage and own /etc/ceph/ceph.conf on the hosts.", 2023-07-14T11:29:14.274 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.274 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.274 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.275 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.275 INFO:teuthology.orchestra.run.smithi040.stdout: "manage_etc_ceph_ceph_conf_hosts": { 2023-07-14T11:29:14.275 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "manage_etc_ceph_ceph_conf_hosts", 2023-07-14T11:29:14.275 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.275 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.276 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.276 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "*", 2023-07-14T11:29:14.276 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.276 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.276 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.277 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf", 2023-07-14T11:29:14.277 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.277 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.277 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.277 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.278 INFO:teuthology.orchestra.run.smithi040.stdout: "max_count_per_host": { 2023-07-14T11:29:14.278 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_count_per_host", 2023-07-14T11:29:14.278 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.278 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.278 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.279 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.279 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.279 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.279 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.279 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of daemons per service per host", 2023-07-14T11:29:14.280 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.280 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.280 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.280 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.280 INFO:teuthology.orchestra.run.smithi040.stdout: "max_osd_draining_count": { 2023-07-14T11:29:14.281 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_osd_draining_count", 2023-07-14T11:29:14.281 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.281 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.281 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.281 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.282 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.282 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.282 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.282 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "max number of osds that will be drained simultaneously when osds are removed", 2023-07-14T11:29:14.282 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.283 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.283 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.283 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.283 INFO:teuthology.orchestra.run.smithi040.stdout: "migration_current": { 2023-07-14T11:29:14.283 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "migration_current", 2023-07-14T11:29:14.284 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.284 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.284 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.284 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.284 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.285 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.285 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.285 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "internal - do not modify", 2023-07-14T11:29:14.285 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.285 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: "mode": { 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mode", 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.286 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.287 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.287 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "root", 2023-07-14T11:29:14.287 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.287 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.287 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.288 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm-package", 2023-07-14T11:29:14.288 INFO:teuthology.orchestra.run.smithi040.stdout: "root" 2023-07-14T11:29:14.288 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.288 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "mode for remote execution of cephadm", 2023-07-14T11:29:14.288 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.289 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.289 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.289 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.289 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_alerts_path": { 2023-07-14T11:29:14.289 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_alerts_path", 2023-07-14T11:29:14.290 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.290 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.290 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.290 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/etc/prometheus/ceph/ceph_default_alerts.yml", 2023-07-14T11:29:14.290 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.291 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.291 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.291 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "location of alerts to include in prometheus deployments", 2023-07-14T11:29:14.291 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.291 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.292 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.292 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.292 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_password": { 2023-07-14T11:29:14.292 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_password", 2023-07-14T11:29:14.292 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.293 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.293 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.293 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.293 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.293 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.294 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.294 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web password", 2023-07-14T11:29:14.294 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.294 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.294 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.295 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.295 INFO:teuthology.orchestra.run.smithi040.stdout: "prometheus_web_user": { 2023-07-14T11:29:14.295 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus_web_user", 2023-07-14T11:29:14.295 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.295 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.296 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.296 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.296 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.296 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.296 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.297 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Prometheus web user", 2023-07-14T11:29:14.297 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.297 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.297 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.298 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.298 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_insecure": { 2023-07-14T11:29:14.298 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_insecure", 2023-07-14T11:29:14.298 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.298 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.299 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.299 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.299 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.299 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.299 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.300 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry is to be considered insecure (no TLS available). Only for development purposes.", 2023-07-14T11:29:14.300 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.300 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.300 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.300 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.301 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_password": { 2023-07-14T11:29:14.301 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_password", 2023-07-14T11:29:14.301 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.301 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.301 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.302 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.302 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.302 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.302 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.303 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository password. Only used for logging into a registry.", 2023-07-14T11:29:14.303 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.303 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.303 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.303 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.304 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_url": { 2023-07-14T11:29:14.304 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_url", 2023-07-14T11:29:14.304 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.304 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.305 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.305 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.305 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.305 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.305 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.306 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Registry url for login purposes. This is not the default registry", 2023-07-14T11:29:14.306 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.306 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.306 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.306 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.307 INFO:teuthology.orchestra.run.smithi040.stdout: "registry_username": { 2023-07-14T11:29:14.307 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "registry_username", 2023-07-14T11:29:14.307 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.307 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.307 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.308 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.308 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.308 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.308 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.308 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Custom repository username. Only used for logging into a registry.", 2023-07-14T11:29:14.309 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.309 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.309 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.309 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.309 INFO:teuthology.orchestra.run.smithi040.stdout: "secure_monitoring_stack": { 2023-07-14T11:29:14.310 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "secure_monitoring_stack", 2023-07-14T11:29:14.310 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.310 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.310 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.310 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.311 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.311 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.311 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.311 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable TLS security for all the monitoring stack daemons", 2023-07-14T11:29:14.311 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.312 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.312 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.312 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.312 INFO:teuthology.orchestra.run.smithi040.stdout: "service_discovery_port": { 2023-07-14T11:29:14.312 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "service_discovery_port", 2023-07-14T11:29:14.313 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.313 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.313 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.313 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8765", 2023-07-14T11:29:14.314 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.314 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.314 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.314 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "cephadm service discovery port", 2023-07-14T11:29:14.314 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.315 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.315 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.315 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.315 INFO:teuthology.orchestra.run.smithi040.stdout: "ssh_config_file": { 2023-07-14T11:29:14.315 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssh_config_file", 2023-07-14T11:29:14.316 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.316 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.316 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.316 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.317 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.317 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.317 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.317 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "customized SSH config file to connect to managed hosts", 2023-07-14T11:29:14.317 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.318 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.318 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.318 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.318 INFO:teuthology.orchestra.run.smithi040.stdout: "use_agent": { 2023-07-14T11:29:14.318 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_agent", 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.319 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.320 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.320 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use cephadm agent on each host to gather and send metadata", 2023-07-14T11:29:14.320 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.320 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.320 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.321 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.321 INFO:teuthology.orchestra.run.smithi040.stdout: "use_repo_digest": { 2023-07-14T11:29:14.321 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "use_repo_digest", 2023-07-14T11:29:14.321 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.321 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.322 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.322 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.322 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.322 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.322 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.323 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Automatically convert image tags to image digest. Make sure all daemons use the same image", 2023-07-14T11:29:14.323 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.323 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.323 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.323 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.324 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_failed_host_check": { 2023-07-14T11:29:14.324 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_failed_host_check", 2023-07-14T11:29:14.324 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.324 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.324 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.325 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.325 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.325 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.325 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.325 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if the host check fails", 2023-07-14T11:29:14.326 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.326 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.326 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.326 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.326 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_daemons": { 2023-07-14T11:29:14.327 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_daemons", 2023-07-14T11:29:14.327 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.327 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.327 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.327 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.328 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.328 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.328 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.328 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected that are not managed by cephadm", 2023-07-14T11:29:14.328 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_on_stray_hosts": { 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_on_stray_hosts", 2023-07-14T11:29:14.329 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.330 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.330 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.330 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.330 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.330 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.331 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.331 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise a health warning if daemons are detected on a host that is not managed by cephadm", 2023-07-14T11:29:14.331 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.331 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.331 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.332 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.332 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.332 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.332 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.332 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crash", 2023-07-14T11:29:14.333 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.333 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.333 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.333 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.333 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.334 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.335 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.335 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.335 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.335 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.335 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.336 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.336 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.336 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.336 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.336 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.337 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.337 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.337 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.337 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.337 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.338 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.338 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.338 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.338 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.338 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.339 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.339 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.339 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.339 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.339 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.340 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.340 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.340 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.340 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.340 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.341 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.341 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.341 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.341 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.341 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.342 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.342 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.342 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.342 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.342 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.343 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.343 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.343 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.343 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.343 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.344 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.344 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.344 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.344 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.344 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.345 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.345 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.345 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.345 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.345 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.346 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.346 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.346 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.346 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.346 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.347 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.347 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.347 INFO:teuthology.orchestra.run.smithi040.stdout: "retain_interval": { 2023-07-14T11:29:14.347 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retain_interval", 2023-07-14T11:29:14.347 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.348 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.348 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.348 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "31536000", 2023-07-14T11:29:14.348 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.349 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.349 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.349 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain crashes before pruning them", 2023-07-14T11:29:14.349 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.349 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.350 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.350 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.350 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_recent_interval": { 2023-07-14T11:29:14.350 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_recent_interval", 2023-07-14T11:29:14.350 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.351 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.351 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.351 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1209600", 2023-07-14T11:29:14.351 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.351 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.352 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.352 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "time interval in which to warn about recent crashes", 2023-07-14T11:29:14.352 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.352 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.352 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.353 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.353 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.353 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.353 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.353 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dashboard", 2023-07-14T11:29:14.354 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.354 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.354 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.354 INFO:teuthology.orchestra.run.smithi040.stdout: "ACCOUNT_LOCKOUT_ATTEMPTS": { 2023-07-14T11:29:14.354 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ACCOUNT_LOCKOUT_ATTEMPTS", 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.355 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.356 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.356 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.356 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.356 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.356 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.357 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.357 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_HOST": { 2023-07-14T11:29:14.357 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_HOST", 2023-07-14T11:29:14.357 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.357 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.358 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.358 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.358 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.358 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.358 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.359 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.359 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.359 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.359 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.359 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.360 INFO:teuthology.orchestra.run.smithi040.stdout: "ALERTMANAGER_API_SSL_VERIFY": { 2023-07-14T11:29:14.360 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ALERTMANAGER_API_SSL_VERIFY", 2023-07-14T11:29:14.360 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.360 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.360 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.361 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.361 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.361 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.361 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.361 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_ENABLED": { 2023-07-14T11:29:14.362 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_ENABLED", 2023-07-14T11:29:14.363 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.363 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.363 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.363 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.363 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.364 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.364 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.364 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.364 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.365 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.365 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.365 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.365 INFO:teuthology.orchestra.run.smithi040.stdout: "AUDIT_API_LOG_PAYLOAD": { 2023-07-14T11:29:14.365 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "AUDIT_API_LOG_PAYLOAD", 2023-07-14T11:29:14.366 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.366 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.366 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.366 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.366 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.367 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.367 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.367 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.367 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.367 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.368 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.368 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.368 INFO:teuthology.orchestra.run.smithi040.stdout: "ENABLE_BROWSABLE_API": { 2023-07-14T11:29:14.368 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ENABLE_BROWSABLE_API", 2023-07-14T11:29:14.368 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.369 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.369 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.369 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.369 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.369 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.370 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.370 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.370 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.370 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.370 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.371 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.371 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_CEPHFS": { 2023-07-14T11:29:14.371 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_CEPHFS", 2023-07-14T11:29:14.371 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.371 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.372 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.372 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.372 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.373 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.373 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.373 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.373 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.373 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.374 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.374 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.374 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_DASHBOARD": { 2023-07-14T11:29:14.374 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_DASHBOARD", 2023-07-14T11:29:14.374 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.375 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.375 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.375 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.375 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.375 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.376 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.376 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.376 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.376 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.377 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.377 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.377 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_ISCSI": { 2023-07-14T11:29:14.377 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_ISCSI", 2023-07-14T11:29:14.377 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.378 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.378 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.378 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.378 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.378 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.379 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.379 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.379 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.379 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.379 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.380 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.380 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_MIRRORING": { 2023-07-14T11:29:14.380 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_MIRRORING", 2023-07-14T11:29:14.380 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.380 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.381 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.381 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.381 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.381 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.381 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.382 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.382 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.382 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.382 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.382 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.383 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_NFS": { 2023-07-14T11:29:14.383 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_NFS", 2023-07-14T11:29:14.383 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.383 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.383 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.384 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.384 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.384 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.384 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.384 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.385 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.385 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.388 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.389 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.389 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RBD": { 2023-07-14T11:29:14.389 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RBD", 2023-07-14T11:29:14.389 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.389 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.390 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.390 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.390 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.390 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.391 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.392 INFO:teuthology.orchestra.run.smithi040.stdout: "FEATURE_TOGGLE_RGW": { 2023-07-14T11:29:14.392 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "FEATURE_TOGGLE_RGW", 2023-07-14T11:29:14.392 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.392 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.393 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.393 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.393 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.393 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.393 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.394 INFO:teuthology.orchestra.run.smithi040.stdout: "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE": { 2023-07-14T11:29:14.395 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE", 2023-07-14T11:29:14.395 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.395 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.395 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.395 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.396 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.396 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.396 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.396 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.397 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.397 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.397 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.397 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.397 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_PASSWORD": { 2023-07-14T11:29:14.398 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_PASSWORD", 2023-07-14T11:29:14.398 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.398 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.398 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.398 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.399 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.399 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.399 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.399 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.400 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.400 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.400 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.400 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.400 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_SSL_VERIFY": { 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_SSL_VERIFY", 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.401 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.402 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.402 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.402 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.402 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.402 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.403 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.403 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.403 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_URL": { 2023-07-14T11:29:14.403 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_URL", 2023-07-14T11:29:14.403 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.404 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.404 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.404 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.404 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.405 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.405 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.405 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.405 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.405 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.406 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.406 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.406 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_API_USERNAME": { 2023-07-14T11:29:14.406 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_API_USERNAME", 2023-07-14T11:29:14.406 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.407 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.407 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.407 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.407 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.407 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.408 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.408 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.408 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.408 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.409 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.409 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.409 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_FRONTEND_API_URL": { 2023-07-14T11:29:14.409 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_FRONTEND_API_URL", 2023-07-14T11:29:14.409 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.410 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.410 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.410 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.410 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.410 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.411 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.411 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.411 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.411 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.411 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.412 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.412 INFO:teuthology.orchestra.run.smithi040.stdout: "GRAFANA_UPDATE_DASHBOARDS": { 2023-07-14T11:29:14.412 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "GRAFANA_UPDATE_DASHBOARDS", 2023-07-14T11:29:14.412 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.412 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.413 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.413 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.413 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.413 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.413 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.414 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.414 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.414 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.414 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.414 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.415 INFO:teuthology.orchestra.run.smithi040.stdout: "ISCSI_API_SSL_VERIFICATION": { 2023-07-14T11:29:14.415 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISCSI_API_SSL_VERIFICATION", 2023-07-14T11:29:14.415 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.415 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.415 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.416 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.416 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.416 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.416 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.416 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.417 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.417 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.417 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.417 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.417 INFO:teuthology.orchestra.run.smithi040.stdout: "ISSUE_TRACKER_API_KEY": { 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ISSUE_TRACKER_API_KEY", 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.418 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.419 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.419 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.419 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.419 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.419 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.420 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.420 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.420 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_HOST": { 2023-07-14T11:29:14.420 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_HOST", 2023-07-14T11:29:14.421 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.421 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.421 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.421 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.421 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.422 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.422 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.422 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.422 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.422 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.423 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.423 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.423 INFO:teuthology.orchestra.run.smithi040.stdout: "PROMETHEUS_API_SSL_VERIFY": { 2023-07-14T11:29:14.423 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PROMETHEUS_API_SSL_VERIFY", 2023-07-14T11:29:14.423 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.424 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.424 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.424 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.424 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.424 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.425 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.425 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.425 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.425 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.425 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.426 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.426 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_COMPLEXITY_ENABLED": { 2023-07-14T11:29:14.426 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_COMPLEXITY_ENABLED", 2023-07-14T11:29:14.426 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.426 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.427 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.427 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.427 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.427 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.427 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.428 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.428 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.428 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.428 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.428 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.429 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED": { 2023-07-14T11:29:14.429 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED", 2023-07-14T11:29:14.429 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.429 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.429 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.430 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.430 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.430 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.430 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.431 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.431 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.431 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.431 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.431 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.432 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_LENGTH_ENABLED": { 2023-07-14T11:29:14.432 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_LENGTH_ENABLED", 2023-07-14T11:29:14.432 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.432 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.433 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.433 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.433 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.433 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.433 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.434 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_OLDPWD_ENABLED": { 2023-07-14T11:29:14.435 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_OLDPWD_ENABLED", 2023-07-14T11:29:14.435 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.435 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.435 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.435 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.436 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.436 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.436 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.436 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.437 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.437 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.437 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.437 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.437 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED": { 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED", 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.438 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.439 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.439 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.439 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.439 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.440 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.440 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.440 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.440 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED": { 2023-07-14T11:29:14.440 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED", 2023-07-14T11:29:14.441 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.441 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.441 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.441 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.441 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.442 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.442 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.442 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.442 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.443 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.443 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.443 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.443 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_CHECK_USERNAME_ENABLED": { 2023-07-14T11:29:14.443 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_CHECK_USERNAME_ENABLED", 2023-07-14T11:29:14.444 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.444 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.444 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.444 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.444 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.445 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.445 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.445 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.445 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.445 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.446 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.446 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.446 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_ENABLED": { 2023-07-14T11:29:14.446 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_ENABLED", 2023-07-14T11:29:14.446 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.447 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.447 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.447 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.447 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.447 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.448 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.448 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.448 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.448 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.448 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.449 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.449 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_EXCLUSION_LIST": { 2023-07-14T11:29:14.449 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_EXCLUSION_LIST", 2023-07-14T11:29:14.449 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.449 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.450 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.450 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps", 2023-07-14T11:29:14.450 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.450 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.451 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.451 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.451 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.451 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.451 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.452 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.452 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_COMPLEXITY": { 2023-07-14T11:29:14.452 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_COMPLEXITY", 2023-07-14T11:29:14.452 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.452 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.453 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.453 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.453 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.453 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.453 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.454 INFO:teuthology.orchestra.run.smithi040.stdout: "PWD_POLICY_MIN_LENGTH": { 2023-07-14T11:29:14.455 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "PWD_POLICY_MIN_LENGTH", 2023-07-14T11:29:14.455 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.455 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.455 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.455 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8", 2023-07-14T11:29:14.456 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.456 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.456 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.456 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.456 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.457 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.457 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.457 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.457 INFO:teuthology.orchestra.run.smithi040.stdout: "REST_REQUESTS_TIMEOUT": { 2023-07-14T11:29:14.457 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "REST_REQUESTS_TIMEOUT", 2023-07-14T11:29:14.458 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.458 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.458 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.458 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "45", 2023-07-14T11:29:14.458 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.459 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.459 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.459 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.459 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.459 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.460 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.460 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.460 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ACCESS_KEY": { 2023-07-14T11:29:14.460 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ACCESS_KEY", 2023-07-14T11:29:14.460 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.461 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.461 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.461 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.461 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.461 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.462 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.462 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.462 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.462 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.462 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_ADMIN_RESOURCE": { 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_ADMIN_RESOURCE", 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.463 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.464 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "admin", 2023-07-14T11:29:14.464 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.464 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.464 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.464 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.465 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.465 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.465 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.465 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.465 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SECRET_KEY": { 2023-07-14T11:29:14.466 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SECRET_KEY", 2023-07-14T11:29:14.466 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.466 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.466 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.466 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.467 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.467 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.467 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.467 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.467 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.468 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.468 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.468 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.468 INFO:teuthology.orchestra.run.smithi040.stdout: "RGW_API_SSL_VERIFY": { 2023-07-14T11:29:14.468 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "RGW_API_SSL_VERIFY", 2023-07-14T11:29:14.469 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.469 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.469 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.469 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.469 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.470 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.470 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.470 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.470 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.470 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_SPAN": { 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_SPAN", 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.471 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.472 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.472 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:14.472 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.472 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.472 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.473 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.473 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.473 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.473 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.473 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.474 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_1": { 2023-07-14T11:29:14.474 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_1", 2023-07-14T11:29:14.474 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.474 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.474 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.475 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.475 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.475 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.475 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.475 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.476 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.476 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.476 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.476 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.476 INFO:teuthology.orchestra.run.smithi040.stdout: "USER_PWD_EXPIRATION_WARNING_2": { 2023-07-14T11:29:14.477 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "USER_PWD_EXPIRATION_WARNING_2", 2023-07-14T11:29:14.477 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.477 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.477 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.477 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:14.478 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.478 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.478 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.478 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.478 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.479 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.479 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.479 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.479 INFO:teuthology.orchestra.run.smithi040.stdout: "cross_origin_url": { 2023-07-14T11:29:14.479 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cross_origin_url", 2023-07-14T11:29:14.480 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.480 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.480 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.480 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.480 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.481 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.481 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.481 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.481 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.481 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: "crt_file": { 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "crt_file", 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.482 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.483 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.483 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.483 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.483 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.483 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.484 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.484 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.484 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.484 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.484 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.485 INFO:teuthology.orchestra.run.smithi040.stdout: "debug": { 2023-07-14T11:29:14.485 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "debug", 2023-07-14T11:29:14.485 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.485 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.485 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.486 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.486 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.486 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.486 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.486 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Enable/disable debug options", 2023-07-14T11:29:14.487 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.487 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.487 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.487 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.487 INFO:teuthology.orchestra.run.smithi040.stdout: "jwt_token_ttl": { 2023-07-14T11:29:14.488 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "jwt_token_ttl", 2023-07-14T11:29:14.488 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.488 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.488 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.488 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "28800", 2023-07-14T11:29:14.489 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.489 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.489 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.489 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.489 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.490 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.490 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.490 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.490 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:14.490 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.491 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.492 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.492 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.492 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.492 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.492 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.493 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.493 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.493 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.493 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.493 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.494 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.494 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.494 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.494 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.494 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.495 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.495 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.495 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.495 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.495 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.496 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.496 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.496 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.496 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.496 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.497 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.498 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.498 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.498 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.498 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.498 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.499 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.499 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.499 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.499 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.500 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.500 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.500 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.500 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.500 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.501 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.501 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.501 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.501 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.501 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.502 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.502 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.502 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.502 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.502 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.503 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.504 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.504 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.504 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.504 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.504 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.505 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.505 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.505 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.505 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.505 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.506 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.506 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.506 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.506 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.506 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.507 INFO:teuthology.orchestra.run.smithi040.stdout: "motd": { 2023-07-14T11:29:14.507 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "motd", 2023-07-14T11:29:14.507 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.507 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.507 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.508 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.508 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.508 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.508 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.508 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "The message of the day", 2023-07-14T11:29:14.509 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.509 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.509 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.509 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.509 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect_resolve_ip_addr": { 2023-07-14T11:29:14.510 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "redirect_resolve_ip_addr", 2023-07-14T11:29:14.510 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.510 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.510 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.510 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.511 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.511 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.511 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.511 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.511 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.512 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.512 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.512 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.512 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:14.512 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:14.513 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.513 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.513 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.513 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:14.513 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.514 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.514 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.514 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.514 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.514 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.515 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.515 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.515 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:14.515 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:14.515 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.516 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.516 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.516 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8080", 2023-07-14T11:29:14.516 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.516 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.517 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.517 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.517 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.517 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.517 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.518 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.519 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.519 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.519 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.519 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.519 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.520 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.520 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.520 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.520 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.520 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl_server_port": { 2023-07-14T11:29:14.521 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl_server_port", 2023-07-14T11:29:14.521 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.521 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.521 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.521 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8443", 2023-07-14T11:29:14.522 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.522 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.522 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.522 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.522 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.523 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.523 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.523 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.523 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:14.523 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:14.524 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.524 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.524 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.524 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "redirect", 2023-07-14T11:29:14.524 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.525 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.525 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.525 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.525 INFO:teuthology.orchestra.run.smithi040.stdout: "redirect" 2023-07-14T11:29:14.525 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.526 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.526 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.526 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.526 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.526 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:14.528 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:14.528 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:14.528 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.528 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.528 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "url_prefix": { 2023-07-14T11:29:14.529 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url_prefix", 2023-07-14T11:29:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.530 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.531 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.531 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.531 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.531 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.531 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.532 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.532 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.532 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.532 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.532 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.533 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "devicehealth", 2023-07-14T11:29:14.533 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.533 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.533 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.533 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_monitoring": { 2023-07-14T11:29:14.534 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_monitoring", 2023-07-14T11:29:14.534 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.534 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.534 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.534 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.535 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.535 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.535 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.535 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "monitor device health metrics", 2023-07-14T11:29:14.535 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.536 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.536 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.536 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.536 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.536 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.537 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.537 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.537 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.537 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.537 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.538 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.538 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.538 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.538 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.538 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.539 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.539 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.539 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.539 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.539 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.540 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.540 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.540 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.540 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.540 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.541 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.542 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.542 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.542 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.542 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.542 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.543 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.543 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.543 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.543 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.543 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.544 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.544 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.544 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.544 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.544 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.545 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.545 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.545 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.545 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.545 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.546 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.546 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.546 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.546 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.546 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.547 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.548 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.548 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.548 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.548 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.548 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.549 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.549 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.549 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.549 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.549 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.550 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.550 INFO:teuthology.orchestra.run.smithi040.stdout: "mark_out_threshold": { 2023-07-14T11:29:14.550 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mark_out_threshold", 2023-07-14T11:29:14.550 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.550 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.551 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.551 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "2419200", 2023-07-14T11:29:14.551 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.551 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.551 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.552 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "automatically mark OSD if it may fail before this long", 2023-07-14T11:29:14.552 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.552 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.552 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.552 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.553 INFO:teuthology.orchestra.run.smithi040.stdout: "pool_name": { 2023-07-14T11:29:14.553 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pool_name", 2023-07-14T11:29:14.553 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.553 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.553 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "device_health_metrics", 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name of pool in which to store device health metrics", 2023-07-14T11:29:14.554 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.555 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.555 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.555 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.555 INFO:teuthology.orchestra.run.smithi040.stdout: "retention_period": { 2023-07-14T11:29:14.555 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "retention_period", 2023-07-14T11:29:14.556 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.556 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.556 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.556 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15552000", 2023-07-14T11:29:14.556 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.557 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.557 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.557 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long to retain device health metrics", 2023-07-14T11:29:14.557 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.557 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.558 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.558 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.558 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_frequency": { 2023-07-14T11:29:14.558 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_frequency", 2023-07-14T11:29:14.558 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.559 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.560 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to scrape device health metrics", 2023-07-14T11:29:14.560 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.560 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.560 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.560 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.561 INFO:teuthology.orchestra.run.smithi040.stdout: "self_heal": { 2023-07-14T11:29:14.561 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "self_heal", 2023-07-14T11:29:14.561 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.561 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.561 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.562 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.562 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.562 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.562 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.562 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "preemptively heal cluster around devices that may fail", 2023-07-14T11:29:14.563 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.563 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.563 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.563 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.563 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:14.564 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:14.564 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.564 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.564 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.564 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:14.565 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.565 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.565 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.565 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how frequently to wake up and check device health", 2023-07-14T11:29:14.565 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: "warn_threshold": { 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "warn_threshold", 2023-07-14T11:29:14.566 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.567 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.567 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.567 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7257600", 2023-07-14T11:29:14.567 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.567 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.568 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.568 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "raise health warning if OSD may fail before this long", 2023-07-14T11:29:14.568 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.568 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.568 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.569 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.569 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.569 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.569 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.569 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "diskprediction_local", 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.570 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.571 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.571 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.571 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.571 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.571 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.572 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.572 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.572 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.572 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.573 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.574 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.574 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.574 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.574 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.574 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.575 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.575 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.575 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.575 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.575 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.576 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.576 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.576 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.576 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.576 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.577 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.577 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.577 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.577 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.577 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.578 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.578 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.578 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.578 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.578 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.579 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.580 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.580 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.580 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.580 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.580 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.581 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.581 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.581 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.581 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.581 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.582 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.582 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.582 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.582 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.582 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.583 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.583 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.583 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.583 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.583 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: "predict_interval": { 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predict_interval", 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.584 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.585 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "86400", 2023-07-14T11:29:14.585 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.585 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.585 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.586 INFO:teuthology.orchestra.run.smithi040.stdout: "predictor_model": { 2023-07-14T11:29:14.587 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "predictor_model", 2023-07-14T11:29:14.587 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.587 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.587 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.587 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "prophetstor", 2023-07-14T11:29:14.588 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.588 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.588 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.588 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.588 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.589 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.589 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.589 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.589 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:14.589 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:14.590 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.590 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.590 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.590 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "600", 2023-07-14T11:29:14.590 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.591 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.591 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.591 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.591 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.591 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.592 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.592 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.592 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.592 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.592 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "influx", 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": false, 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "influxdb python module not found", 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "batch_size": { 2023-07-14T11:29:14.593 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "batch_size", 2023-07-14T11:29:14.594 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.594 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.594 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.594 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5000", 2023-07-14T11:29:14.594 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.595 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.595 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.595 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How big batches of data points should be when sending to InfluxDB.", 2023-07-14T11:29:14.595 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.595 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.596 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.596 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.596 INFO:teuthology.orchestra.run.smithi040.stdout: "database": { 2023-07-14T11:29:14.596 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "database", 2023-07-14T11:29:14.596 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.597 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.597 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.597 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "ceph", 2023-07-14T11:29:14.597 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.597 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.598 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.598 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.", 2023-07-14T11:29:14.598 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.598 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.598 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: "hostname": { 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "hostname", 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.599 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.600 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.600 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.600 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.600 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.601 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server hostname", 2023-07-14T11:29:14.601 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.601 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.601 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.601 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.602 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:14.602 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:14.602 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.602 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.602 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.603 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "30", 2023-07-14T11:29:14.603 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "5", 2023-07-14T11:29:14.603 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.603 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.603 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Time between reports to InfluxDB. Default 30 seconds.", 2023-07-14T11:29:14.604 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.604 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.604 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.604 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.604 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.605 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.606 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.606 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.606 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.606 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.606 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.607 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.607 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.607 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.607 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.607 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.608 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.608 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.608 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.608 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.608 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.609 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.609 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.609 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.609 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.609 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.610 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.610 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.610 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.610 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.610 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.611 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.611 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.611 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.611 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.611 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.612 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.612 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.612 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.612 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.612 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.613 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.613 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.613 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.613 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.613 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.614 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.614 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.614 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.614 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.614 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.615 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.615 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.615 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.615 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.615 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.616 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.616 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.616 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.616 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.616 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.617 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.617 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.617 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.617 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.617 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.618 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.618 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.618 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.618 INFO:teuthology.orchestra.run.smithi040.stdout: "password": { 2023-07-14T11:29:14.618 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "password", 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.619 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.620 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.620 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "password of InfluxDB server user", 2023-07-14T11:29:14.620 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.620 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.621 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.621 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.621 INFO:teuthology.orchestra.run.smithi040.stdout: "port": { 2023-07-14T11:29:14.621 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "port", 2023-07-14T11:29:14.621 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.622 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.622 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.622 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "8086", 2023-07-14T11:29:14.622 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.623 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.623 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.623 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "InfluxDB server port", 2023-07-14T11:29:14.623 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.623 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.624 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.624 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.624 INFO:teuthology.orchestra.run.smithi040.stdout: "ssl": { 2023-07-14T11:29:14.624 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ssl", 2023-07-14T11:29:14.625 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.625 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.625 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.625 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "false", 2023-07-14T11:29:14.626 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.626 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.626 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.626 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Use https connection for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:14.627 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.627 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.627 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.627 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.628 INFO:teuthology.orchestra.run.smithi040.stdout: "threads": { 2023-07-14T11:29:14.628 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threads", 2023-07-14T11:29:14.628 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.628 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.629 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.629 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:14.629 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:14.630 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "32", 2023-07-14T11:29:14.630 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.630 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "How many worker threads should be spawned for sending data to InfluxDB.", 2023-07-14T11:29:14.630 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.631 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.631 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.631 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.631 INFO:teuthology.orchestra.run.smithi040.stdout: "username": { 2023-07-14T11:29:14.632 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "username", 2023-07-14T11:29:14.632 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.632 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.632 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.633 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.633 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.633 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.633 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.633 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "username of InfluxDB server user", 2023-07-14T11:29:14.634 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.634 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.634 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.634 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.634 INFO:teuthology.orchestra.run.smithi040.stdout: "verify_ssl": { 2023-07-14T11:29:14.635 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "verify_ssl", 2023-07-14T11:29:14.635 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.635 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.635 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.635 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "true", 2023-07-14T11:29:14.636 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.636 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.636 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.636 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Verify https cert for InfluxDB server. Use \"true\" or \"false\".", 2023-07-14T11:29:14.636 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.637 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.637 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.637 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.637 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.637 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.638 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.638 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "insights", 2023-07-14T11:29:14.638 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.638 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.638 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.639 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.640 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.640 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.640 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.640 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.640 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.641 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.641 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.641 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.641 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.641 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.642 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.642 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.642 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.642 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.642 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.643 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.643 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.643 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.643 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.643 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.644 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.644 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.644 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.644 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.644 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.645 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.645 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.645 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.645 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.645 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.646 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.646 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.646 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.646 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.646 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.647 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.647 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.647 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.647 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.647 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.648 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.648 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.648 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.648 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.648 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.649 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.649 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.649 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.649 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.649 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.650 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.650 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.650 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.650 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.650 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.651 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.651 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.651 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.651 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.651 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.652 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.652 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.652 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.652 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.652 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.653 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.653 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.653 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "iostat", 2023-07-14T11:29:14.653 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.653 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.654 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.654 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.654 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.654 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.654 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.655 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.655 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.655 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.655 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.655 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.656 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.656 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.656 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.656 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.656 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.657 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.657 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.657 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.657 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.657 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.658 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.658 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.658 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.658 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.658 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.659 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.660 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.660 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.660 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.660 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.660 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.661 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.661 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.661 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.661 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.661 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.662 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.662 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.662 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.662 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.662 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.663 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.664 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.664 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.664 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.664 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.665 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.666 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.666 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.666 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.666 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.666 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.667 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.667 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.667 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.667 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.667 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.668 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.668 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.668 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "k8sevents", 2023-07-14T11:29:14.668 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.668 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.669 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.669 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_event_retention_days": { 2023-07-14T11:29:14.669 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "ceph_event_retention_days", 2023-07-14T11:29:14.669 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.670 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.670 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.670 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "7", 2023-07-14T11:29:14.670 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.670 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.671 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.671 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Days to hold ceph event information within local cache", 2023-07-14T11:29:14.671 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.671 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.671 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.672 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.672 INFO:teuthology.orchestra.run.smithi040.stdout: "config_check_secs": { 2023-07-14T11:29:14.672 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "config_check_secs", 2023-07-14T11:29:14.672 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.673 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.673 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.673 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.673 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "10", 2023-07-14T11:29:14.673 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.674 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.674 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval (secs) to check for cluster configuration changes", 2023-07-14T11:29:14.674 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.674 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.674 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.675 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.675 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.675 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.675 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.675 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.676 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.676 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.676 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.676 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.676 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.677 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.677 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.677 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.677 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.677 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.678 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.678 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.678 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.678 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.678 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.679 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.680 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.680 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.680 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.680 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.680 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.681 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.681 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.681 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.681 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.681 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.682 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.682 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.682 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.682 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.682 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.683 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.683 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.683 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.683 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.683 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.684 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.684 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.684 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.684 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.684 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.685 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.685 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.685 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.685 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.685 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.686 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.686 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.686 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.686 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.686 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.687 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.688 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.688 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.688 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.688 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.688 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.689 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.689 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.689 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "localpool", 2023-07-14T11:29:14.689 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "failure_domain": { 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "failure_domain", 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.690 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.691 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.691 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "host", 2023-07-14T11:29:14.691 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.691 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.691 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.692 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "failure domain for any created local pool", 2023-07-14T11:29:14.692 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "what failure domain we should separate data replicas across.", 2023-07-14T11:29:14.692 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.692 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.692 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.693 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.693 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.693 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.693 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.693 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.694 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.694 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.694 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.694 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.694 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.695 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.695 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.695 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.695 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.695 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.696 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.696 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.696 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.696 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.696 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.697 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.697 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.697 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.697 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.698 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.698 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.698 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.698 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.698 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.699 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.699 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.699 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.699 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.699 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.700 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.700 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.700 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.700 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.700 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.701 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.701 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.701 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.701 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.701 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.702 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.702 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.702 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.702 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.702 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.703 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.703 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.703 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.703 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.703 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.704 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.704 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.704 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.704 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.704 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.705 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.705 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.705 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.705 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.705 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.706 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.706 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.706 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.706 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.706 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: "min_size": { 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "min_size", 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.707 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.708 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.708 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.708 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.708 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.708 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default min_size for any created local pool", 2023-07-14T11:29:14.709 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "value to set min_size to (unchanged from Ceph's default if this option is not set)", 2023-07-14T11:29:14.709 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.709 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.709 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.709 INFO:teuthology.orchestra.run.smithi040.stdout: "num_rep": { 2023-07-14T11:29:14.710 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "num_rep", 2023-07-14T11:29:14.710 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.710 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.710 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.710 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3", 2023-07-14T11:29:14.711 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.711 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.711 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.711 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default replica count for any created local pool", 2023-07-14T11:29:14.711 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.712 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.712 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.712 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.712 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_num": { 2023-07-14T11:29:14.712 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_num", 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "128", 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.713 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.714 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.714 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "default pg_num for any created local pool", 2023-07-14T11:29:14.714 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.714 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.714 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.715 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.715 INFO:teuthology.orchestra.run.smithi040.stdout: "prefix": { 2023-07-14T11:29:14.715 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prefix", 2023-07-14T11:29:14.715 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.715 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.716 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.716 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.716 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.716 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.716 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.717 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "name prefix for any created local pool", 2023-07-14T11:29:14.717 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.717 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.717 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: "subtree": { 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "subtree", 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.718 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.719 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "rack", 2023-07-14T11:29:14.719 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.719 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.719 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.719 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "CRUSH level for which to create a local pool", 2023-07-14T11:29:14.720 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "which CRUSH subtree type the module should create a pool for.", 2023-07-14T11:29:14.720 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.720 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.720 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.720 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.721 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.721 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.721 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mds_autoscaler", 2023-07-14T11:29:14.721 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.721 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.722 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.723 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.723 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.723 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.723 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.723 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.724 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.724 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.724 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.724 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.724 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.725 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.725 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.725 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.725 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.725 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.726 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.726 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.726 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.726 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.726 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.727 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.727 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.727 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.727 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.727 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.728 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.728 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.728 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.728 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.728 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.729 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.729 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.729 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.729 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.729 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.730 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.730 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.730 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.730 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.730 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.731 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.732 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.732 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.732 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.732 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.732 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.733 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.733 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.733 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.733 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.733 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.734 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.734 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.734 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.734 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.735 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.735 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.735 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.735 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.735 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.736 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.736 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.736 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.736 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirroring", 2023-07-14T11:29:14.736 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.737 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.737 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.737 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.737 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.737 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.738 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.738 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.738 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.738 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.738 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.739 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.739 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.739 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.739 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.739 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.740 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.740 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.740 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.740 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.740 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.741 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.741 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.741 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.741 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.741 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.742 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.742 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.742 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.742 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.742 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.743 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.743 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.743 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.744 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.744 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.744 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.744 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.744 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.745 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.745 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.745 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.745 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.745 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.746 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.746 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.746 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.746 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.747 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.747 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.747 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.747 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.748 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.748 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.748 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.748 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.748 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.749 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.749 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.749 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.749 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.750 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.750 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.750 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.750 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.750 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.751 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.751 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.751 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.751 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.751 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.752 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.752 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.752 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.752 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.752 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.753 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "nfs", 2023-07-14T11:29:14.753 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.753 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.753 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.753 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.754 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.754 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.754 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.754 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.754 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.755 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.755 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.755 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.755 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.755 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.756 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.756 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.756 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.756 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.756 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.757 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.758 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.758 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.758 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.758 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.759 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.759 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.759 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.759 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.759 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.760 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.760 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.760 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.760 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.761 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.761 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.761 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.761 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.761 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.762 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.762 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.762 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.762 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.763 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.763 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.763 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.763 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.764 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.764 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.764 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.764 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.764 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.765 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.765 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.765 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.765 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.765 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.766 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.766 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.766 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.766 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.766 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.767 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.767 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.767 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.767 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.767 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.768 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.768 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.768 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.768 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.768 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.769 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:14.769 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.769 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.769 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.769 INFO:teuthology.orchestra.run.smithi040.stdout: "fail_fs": { 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "fail_fs", 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.770 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.771 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.771 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.771 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Fail filesystem for rapid multi-rank mds upgrade", 2023-07-14T11:29:14.771 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.771 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.772 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.772 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.772 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.772 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.772 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.773 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.773 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.773 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.773 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.773 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.774 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.774 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.774 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.774 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.775 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.776 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.776 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.776 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.776 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.776 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.777 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.777 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.777 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.777 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.777 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.778 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.778 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.778 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.778 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.778 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.779 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.779 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.779 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.779 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.779 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.780 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.780 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.780 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.780 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.780 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.781 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.781 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.781 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.781 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.781 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.782 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.782 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.782 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.782 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.782 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.783 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.783 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.783 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.783 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.783 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.784 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.784 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.784 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.784 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.784 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.785 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.785 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.785 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.785 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.785 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.786 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.786 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.786 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator": { 2023-07-14T11:29:14.786 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "orchestrator", 2023-07-14T11:29:14.786 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.787 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.788 INFO:teuthology.orchestra.run.smithi040.stdout: "cephadm", 2023-07-14T11:29:14.788 INFO:teuthology.orchestra.run.smithi040.stdout: "rook", 2023-07-14T11:29:14.788 INFO:teuthology.orchestra.run.smithi040.stdout: "test_orchestrator" 2023-07-14T11:29:14.788 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.788 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Orchestrator backend", 2023-07-14T11:29:14.789 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.789 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.789 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.789 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.789 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.790 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.790 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.790 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_perf_query", 2023-07-14T11:29:14.790 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.791 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.791 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.791 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.791 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.791 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.792 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.792 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.792 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.792 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.792 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.793 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.794 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.794 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.794 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.794 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.794 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.795 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.796 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.797 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.797 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.797 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.797 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.798 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.798 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.798 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.798 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.798 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.799 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.799 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.799 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.799 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.799 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.800 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.800 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.800 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.800 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.800 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.801 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.801 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.801 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.801 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.801 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.802 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.802 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.802 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.802 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.802 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.803 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.803 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.803 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.803 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.803 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.804 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.804 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.804 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.804 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.804 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.805 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.805 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.805 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.805 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.805 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.806 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "osd_support", 2023-07-14T11:29:14.806 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.806 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.806 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.806 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.807 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.808 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.808 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.808 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.808 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.808 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.809 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.809 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.809 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.809 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.809 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.810 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.810 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.810 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.810 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.810 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.811 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.811 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.811 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.811 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.811 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.812 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.812 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.812 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.812 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.812 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.813 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.813 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.813 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.813 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.813 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.814 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.814 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.814 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.814 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.814 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.815 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.815 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.815 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.815 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.815 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.816 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.816 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.816 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.816 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.816 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.817 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.817 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.817 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.817 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.817 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.818 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.818 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.818 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.818 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.818 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.819 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.820 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.820 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.820 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.820 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.820 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.821 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "pg_autoscaler", 2023-07-14T11:29:14.821 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.821 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.821 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.821 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.822 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.822 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.822 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.822 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.822 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.823 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.823 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.823 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.823 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.823 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.824 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.825 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.825 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.825 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.825 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.825 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.826 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.826 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.826 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.826 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.827 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.827 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.827 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.827 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.827 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.828 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.828 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.828 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.828 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.829 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.829 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.829 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.829 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.829 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.830 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.830 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.830 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.830 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.830 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.831 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.831 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.831 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.831 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.831 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.832 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.832 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.832 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.832 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.832 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.833 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.833 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.833 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.833 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.833 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.834 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.834 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.834 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.834 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.834 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.835 INFO:teuthology.orchestra.run.smithi040.stdout: "noautoscale": { 2023-07-14T11:29:14.836 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "noautoscale", 2023-07-14T11:29:14.836 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.836 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.836 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.836 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.837 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.837 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.837 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.837 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "global autoscale flag", 2023-07-14T11:29:14.837 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Option to turn on/off the autoscaler for all pools", 2023-07-14T11:29:14.838 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.838 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.838 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.838 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:14.838 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:14.839 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.839 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.839 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.839 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:14.839 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.840 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.840 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.840 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.840 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.840 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.841 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.841 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.841 INFO:teuthology.orchestra.run.smithi040.stdout: "threshold": { 2023-07-14T11:29:14.841 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "threshold", 2023-07-14T11:29:14.841 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.842 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.842 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.842 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "3.0", 2023-07-14T11:29:14.842 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1.0", 2023-07-14T11:29:14.842 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "scaling threshold", 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0", 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.843 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.844 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.844 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.844 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.844 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "progress", 2023-07-14T11:29:14.845 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.845 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.845 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.845 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_pg_recovery_event": { 2023-07-14T11:29:14.845 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_pg_recovery_event", 2023-07-14T11:29:14.846 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.846 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.846 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.846 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.846 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.847 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.847 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.847 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow the module to show pg recovery progress", 2023-07-14T11:29:14.847 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.847 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.848 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.848 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.848 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:14.848 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:14.848 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.849 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.849 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.849 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.849 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.849 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.850 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.850 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.850 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.850 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.850 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.851 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.852 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.852 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.852 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.852 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.853 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.854 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.854 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.854 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.854 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.854 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.855 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.855 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.855 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.855 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.855 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.856 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.856 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.856 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.856 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.856 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.857 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.857 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.857 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.857 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.857 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.858 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.859 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.859 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.859 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.859 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.859 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.860 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.860 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.860 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.860 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.860 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.861 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.861 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.861 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.861 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.861 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.862 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.862 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.862 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.862 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.863 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.863 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.863 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.863 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.863 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.864 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.864 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.864 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.864 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.864 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.865 INFO:teuthology.orchestra.run.smithi040.stdout: "max_completed_events": { 2023-07-14T11:29:14.865 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_completed_events", 2023-07-14T11:29:14.865 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.865 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.865 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.866 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "50", 2023-07-14T11:29:14.866 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.866 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.866 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.866 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "number of past completed events to remember", 2023-07-14T11:29:14.867 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.867 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.867 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.867 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.867 INFO:teuthology.orchestra.run.smithi040.stdout: "sleep_interval": { 2023-07-14T11:29:14.868 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "sleep_interval", 2023-07-14T11:29:14.868 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:14.868 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.868 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.869 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "5", 2023-07-14T11:29:14.869 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.869 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.869 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.869 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "how long the module is going to sleep", 2023-07-14T11:29:14.870 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.870 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.870 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.870 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.870 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.871 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.871 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.871 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "prometheus", 2023-07-14T11:29:14.871 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.871 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.872 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.872 INFO:teuthology.orchestra.run.smithi040.stdout: "cache": { 2023-07-14T11:29:14.872 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "cache", 2023-07-14T11:29:14.872 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.872 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.873 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.873 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.873 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.873 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.873 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.874 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.874 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.874 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.874 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.874 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.875 INFO:teuthology.orchestra.run.smithi040.stdout: "exclude_perf_counters": { 2023-07-14T11:29:14.875 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "exclude_perf_counters", 2023-07-14T11:29:14.875 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.875 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.875 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.876 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.876 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.876 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.876 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.877 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Do not include perf-counters in the metrics output", 2023-07-14T11:29:14.877 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.", 2023-07-14T11:29:14.877 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.877 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.877 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.878 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.878 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.878 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.878 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.878 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.879 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.879 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.879 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.879 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.879 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.880 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.880 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.880 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.880 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.880 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.881 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.881 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.881 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.881 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.881 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.882 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.882 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.882 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.882 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.882 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.883 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.883 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.883 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.883 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.883 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.884 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.884 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.884 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.884 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.884 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.885 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.885 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.885 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.885 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.885 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.886 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.887 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.887 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.887 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.887 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.887 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.888 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.888 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.888 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.888 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.888 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.889 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.889 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.889 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.889 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.890 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.891 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.891 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.891 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.891 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.891 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools": { 2023-07-14T11:29:14.892 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools", 2023-07-14T11:29:14.892 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.892 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.892 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.892 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.893 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.893 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.893 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.893 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.893 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_stats_pools_refresh_interval": { 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_stats_pools_refresh_interval", 2023-07-14T11:29:14.894 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.895 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.895 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.895 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300", 2023-07-14T11:29:14.895 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.895 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.896 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.896 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.896 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.896 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.896 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.897 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.897 INFO:teuthology.orchestra.run.smithi040.stdout: "scrape_interval": { 2023-07-14T11:29:14.897 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "scrape_interval", 2023-07-14T11:29:14.897 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.897 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.898 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.898 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15.0", 2023-07-14T11:29:14.898 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.898 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.898 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.899 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.899 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.899 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.899 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.899 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.900 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:14.900 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:14.900 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.900 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.900 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "::", 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the IPv4 or IPv6 address on which the module listens for HTTP requests", 2023-07-14T11:29:14.901 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.902 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.902 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.902 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.902 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:14.902 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:14.903 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.903 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.903 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.903 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "9283", 2023-07-14T11:29:14.903 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "the port on which the module listens for HTTP requests", 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.904 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.905 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.905 INFO:teuthology.orchestra.run.smithi040.stdout: "stale_cache_strategy": { 2023-07-14T11:29:14.905 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stale_cache_strategy", 2023-07-14T11:29:14.905 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.905 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.906 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.906 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "log", 2023-07-14T11:29:14.906 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.906 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.906 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.907 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.907 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.907 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.907 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.907 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.908 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_behaviour": { 2023-07-14T11:29:14.908 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_behaviour", 2023-07-14T11:29:14.908 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.908 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.908 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.909 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "default", 2023-07-14T11:29:14.909 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.909 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.909 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.909 INFO:teuthology.orchestra.run.smithi040.stdout: "default", 2023-07-14T11:29:14.910 INFO:teuthology.orchestra.run.smithi040.stdout: "error" 2023-07-14T11:29:14.910 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.910 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.910 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.910 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.911 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.911 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.911 INFO:teuthology.orchestra.run.smithi040.stdout: "standby_error_status_code": { 2023-07-14T11:29:14.911 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "standby_error_status_code", 2023-07-14T11:29:14.911 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.912 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.912 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.912 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "500", 2023-07-14T11:29:14.912 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "400", 2023-07-14T11:29:14.912 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "599", 2023-07-14T11:29:14.913 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.913 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.913 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.913 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.913 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_support", 2023-07-14T11:29:14.914 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.915 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.915 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.915 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.915 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.915 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.916 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.916 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.916 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.916 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.916 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.917 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.917 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.917 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.917 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.917 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.918 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.918 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.918 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.918 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.918 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.920 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.920 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.920 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.920 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.920 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.921 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.921 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.921 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.921 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.921 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.922 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.922 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.922 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.922 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.922 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.923 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.924 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.924 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.924 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.924 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.924 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.925 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.925 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.925 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.925 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.925 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.926 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.926 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.926 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.926 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.926 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.927 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.928 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.928 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.928 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.928 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.928 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.929 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_snap_create": { 2023-07-14T11:29:14.929 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_snap_create", 2023-07-14T11:29:14.929 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:14.929 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.929 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.930 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10", 2023-07-14T11:29:14.930 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.930 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.930 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.930 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.931 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.931 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.931 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.931 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.931 INFO:teuthology.orchestra.run.smithi040.stdout: "mirror_snapshot_schedule": { 2023-07-14T11:29:14.932 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "mirror_snapshot_schedule", 2023-07-14T11:29:14.932 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.932 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.932 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.932 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.933 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.933 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.933 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.933 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.933 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.934 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.934 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.934 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.934 INFO:teuthology.orchestra.run.smithi040.stdout: "trash_purge_schedule": { 2023-07-14T11:29:14.934 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "trash_purge_schedule", 2023-07-14T11:29:14.935 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.935 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.935 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.935 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.935 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.936 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.936 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.936 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.936 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.936 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.937 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.937 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.937 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.937 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.937 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "restful", 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "enable_auth": { 2023-07-14T11:29:14.938 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enable_auth", 2023-07-14T11:29:14.939 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.939 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.939 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.939 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:14.939 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.940 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.940 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.940 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.940 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.940 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.941 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.941 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.941 INFO:teuthology.orchestra.run.smithi040.stdout: "key_file": { 2023-07-14T11:29:14.941 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "key_file", 2023-07-14T11:29:14.941 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.942 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.942 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.942 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.942 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.942 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.943 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.944 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.944 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.944 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.944 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.945 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.945 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.945 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.945 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.945 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.946 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.947 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.947 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.947 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.947 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.947 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.948 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.948 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.948 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.948 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.948 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.949 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.949 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.949 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.949 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.949 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.950 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.950 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.950 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.950 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.950 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.951 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.951 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.951 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.951 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.951 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.952 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.952 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.952 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.952 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.952 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.953 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.953 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.953 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.953 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.953 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.954 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.954 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.954 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.954 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.954 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.955 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.955 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.955 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.955 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.955 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.956 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.956 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.956 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.956 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.957 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.957 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.957 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.957 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.957 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.958 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.958 INFO:teuthology.orchestra.run.smithi040.stdout: "server_addr": { 2023-07-14T11:29:14.958 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_addr", 2023-07-14T11:29:14.958 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.958 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.959 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.959 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.959 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.959 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.959 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.960 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.960 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.960 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.960 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.960 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.961 INFO:teuthology.orchestra.run.smithi040.stdout: "server_port": { 2023-07-14T11:29:14.961 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "server_port", 2023-07-14T11:29:14.961 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.961 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.961 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.962 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.963 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.963 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.963 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.963 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.964 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.964 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.964 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rgw", 2023-07-14T11:29:14.964 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.964 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.965 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.965 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.965 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.965 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.965 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.966 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.967 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.967 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.967 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.967 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.967 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.968 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.968 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.968 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.968 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.968 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.969 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.969 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.969 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.969 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.969 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.970 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.970 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.970 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.970 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.970 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.971 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.971 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.971 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.971 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.971 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.972 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.972 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.972 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.972 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.973 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.973 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.973 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.973 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.973 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.974 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.975 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.975 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.975 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.975 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.975 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.976 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.976 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.976 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.976 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.976 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.977 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.977 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.977 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.977 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.977 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.978 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.978 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.978 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.978 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.978 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rook", 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:14.979 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:14.980 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:14.980 INFO:teuthology.orchestra.run.smithi040.stdout: "drive_group_interval": { 2023-07-14T11:29:14.980 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "drive_group_interval", 2023-07-14T11:29:14.980 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:14.980 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.981 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.981 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "300.0", 2023-07-14T11:29:14.981 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.981 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.981 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.982 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "interval in seconds between re-application of applied drive_groups", 2023-07-14T11:29:14.982 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.982 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.982 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.982 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.983 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:14.983 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:14.983 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.983 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.983 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.984 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.985 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.985 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.985 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.985 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.985 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.986 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.986 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.986 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.986 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.986 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.987 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:14.987 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:14.987 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.987 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.988 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.988 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.988 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.988 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.988 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.989 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.989 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.989 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.989 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.989 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.990 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:14.990 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:14.990 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.990 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.990 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:14.991 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:14.992 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:14.992 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:14.992 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:14.992 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:14.992 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:14.993 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.993 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.993 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.993 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.993 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.994 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:14.994 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:14.994 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:14.994 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.994 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:14.995 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:14.995 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.995 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.995 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.995 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:14.996 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.996 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.996 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.996 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.996 INFO:teuthology.orchestra.run.smithi040.stdout: "storage_class": { 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "storage_class", 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "local", 2023-07-14T11:29:14.997 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:14.998 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:14.998 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:14.998 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "storage class name for LSO-discovered PVs", 2023-07-14T11:29:14.998 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:14.998 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:14.999 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:14.999 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.999 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:14.999 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:14.999 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.000 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "selftest", 2023-07-14T11:29:15.000 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.000 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.000 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.000 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.001 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.001 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.001 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.001 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.001 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.002 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.002 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.002 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.002 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.002 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.003 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.003 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.003 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.003 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.003 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.004 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.004 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.004 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.004 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.004 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.005 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.005 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.005 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.005 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.005 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.006 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.006 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.006 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.006 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.006 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.007 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.007 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.007 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.007 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.007 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.008 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.008 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.008 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.008 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.008 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.009 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.010 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.010 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.010 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.010 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.010 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.011 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.011 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.011 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.011 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.011 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.012 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.012 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.012 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.012 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.012 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.013 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.013 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.013 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.013 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.013 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: "roption1": { 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption1", 2023-07-14T11:29:15.014 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.015 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.015 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.015 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.015 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.015 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.016 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.016 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.016 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.016 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.016 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.017 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.017 INFO:teuthology.orchestra.run.smithi040.stdout: "roption2": { 2023-07-14T11:29:15.017 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "roption2", 2023-07-14T11:29:15.017 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.017 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "xyz", 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.018 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.019 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.019 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.019 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.019 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.019 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption1": { 2023-07-14T11:29:15.020 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption1", 2023-07-14T11:29:15.020 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.020 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.020 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.020 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.021 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption2": { 2023-07-14T11:29:15.022 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption2", 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.023 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.024 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.025 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.025 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption3": { 2023-07-14T11:29:15.025 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption3", 2023-07-14T11:29:15.025 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "float", 2023-07-14T11:29:15.025 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.026 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.026 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.026 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.026 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.026 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.027 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.027 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.027 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.027 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.027 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.028 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption4": { 2023-07-14T11:29:15.028 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption4", 2023-07-14T11:29:15.028 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.028 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.028 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.029 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.030 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption5": { 2023-07-14T11:29:15.031 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption5", 2023-07-14T11:29:15.031 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.031 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.031 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.031 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.032 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.032 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.032 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.032 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.032 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.033 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.033 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.033 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.033 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption6": { 2023-07-14T11:29:15.033 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption6", 2023-07-14T11:29:15.034 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.034 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.034 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.034 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:15.034 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.035 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.036 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.036 INFO:teuthology.orchestra.run.smithi040.stdout: "rwoption7": { 2023-07-14T11:29:15.036 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rwoption7", 2023-07-14T11:29:15.036 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.036 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.037 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.037 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.037 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "1", 2023-07-14T11:29:15.037 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "42", 2023-07-14T11:29:15.037 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.038 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.038 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.038 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.038 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.038 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.039 INFO:teuthology.orchestra.run.smithi040.stdout: "testkey": { 2023-07-14T11:29:15.039 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testkey", 2023-07-14T11:29:15.039 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.039 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.039 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.040 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.040 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.040 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.040 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.040 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.041 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.041 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.041 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.041 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.042 INFO:teuthology.orchestra.run.smithi040.stdout: "testlkey": { 2023-07-14T11:29:15.042 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testlkey", 2023-07-14T11:29:15.042 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.042 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.042 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.043 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.043 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.043 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.043 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.043 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.044 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.044 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.044 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.044 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.044 INFO:teuthology.orchestra.run.smithi040.stdout: "testnewline": { 2023-07-14T11:29:15.045 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "testnewline", 2023-07-14T11:29:15.045 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.045 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.045 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.045 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.046 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.046 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.046 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.046 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.046 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.047 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.047 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.047 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.047 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.047 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.048 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.048 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snap_schedule", 2023-07-14T11:29:15.048 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.048 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.048 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.049 INFO:teuthology.orchestra.run.smithi040.stdout: "allow_m_granularity": { 2023-07-14T11:29:15.049 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "allow_m_granularity", 2023-07-14T11:29:15.049 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.049 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.050 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.050 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.050 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.050 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.051 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.051 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "allow minute scheduled snapshots", 2023-07-14T11:29:15.051 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.051 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.051 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.052 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.052 INFO:teuthology.orchestra.run.smithi040.stdout: "dump_on_update": { 2023-07-14T11:29:15.052 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "dump_on_update", 2023-07-14T11:29:15.052 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.052 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.053 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.053 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.053 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.053 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.053 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.054 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "dump database to debug log on update", 2023-07-14T11:29:15.054 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.054 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.054 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.054 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.055 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.055 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.055 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.055 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.055 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.056 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.056 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.056 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.056 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.056 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.057 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.057 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.057 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.057 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.057 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.058 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.058 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.058 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.058 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.058 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.059 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.059 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.059 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.059 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.059 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.060 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.061 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.062 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.062 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.062 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.062 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.063 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.063 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.063 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.063 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.063 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.064 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.064 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.064 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.064 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.065 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.065 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.065 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.065 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.065 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.066 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.066 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.066 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.066 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.066 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.067 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.067 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.067 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.067 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.067 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.068 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.068 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.068 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.068 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.068 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.069 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.069 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.069 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.069 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "stats", 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.070 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.071 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.071 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.071 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.071 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.071 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.072 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.072 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.072 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.072 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.073 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.073 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.073 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.073 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.073 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.074 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.074 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.074 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.074 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.074 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.075 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.075 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.075 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.075 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.075 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.076 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.076 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.076 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.076 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.076 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.077 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.077 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.077 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.077 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.078 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.079 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.079 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.079 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.079 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.079 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.080 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.080 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.080 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.080 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.080 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.081 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.081 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.081 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.081 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.081 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.082 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.082 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.082 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.082 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.082 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.083 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.083 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.083 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.083 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.083 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.084 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.084 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.084 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.084 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.084 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.085 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.085 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.085 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.085 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "status", 2023-07-14T11:29:15.085 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.086 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.086 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.086 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.086 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.086 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.087 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.088 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.088 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.088 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.088 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.088 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.089 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.089 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.089 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.089 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.089 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.090 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.090 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.090 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.090 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.090 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.091 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.091 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.091 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.091 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.091 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.092 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.092 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.092 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.092 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.093 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.094 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.094 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.094 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.094 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.094 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.095 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.095 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.095 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.095 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.095 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.096 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.096 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.096 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.096 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.096 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.097 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.097 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.097 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.097 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.097 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.098 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.098 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.098 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.098 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.098 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.099 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.099 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.099 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.099 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.099 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.100 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.100 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.100 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.100 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.100 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telegraf", 2023-07-14T11:29:15.101 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.101 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.101 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.101 INFO:teuthology.orchestra.run.smithi040.stdout: "address": { 2023-07-14T11:29:15.101 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "address", 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "unixgram:///tmp/telegraf.sock", 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.102 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.103 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.103 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.103 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.103 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.103 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.104 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.104 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:15.104 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:15.104 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:15.104 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.105 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.105 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "15", 2023-07-14T11:29:15.105 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.105 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.105 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.106 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.107 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.107 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.107 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.107 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.107 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.108 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.108 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.108 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.108 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.108 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.109 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.109 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.109 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.109 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.109 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.110 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.110 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.110 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.110 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.110 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.111 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.111 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.111 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.111 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.111 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.112 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.112 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.112 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.112 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.112 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.113 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.113 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.113 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.113 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.113 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.114 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.115 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.115 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.115 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.115 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.115 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.116 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.116 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.116 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.116 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.116 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.117 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.117 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.117 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.117 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.117 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.118 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.118 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.118 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.118 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.118 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.119 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.119 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.119 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.119 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.119 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.120 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.120 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.120 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.120 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.120 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "telemetry", 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.121 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_basic": { 2023-07-14T11:29:15.122 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_basic", 2023-07-14T11:29:15.122 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.122 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.122 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.122 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:15.123 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.123 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.123 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.123 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share basic cluster information (size, version)", 2023-07-14T11:29:15.123 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.124 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.124 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.124 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.124 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_crash": { 2023-07-14T11:29:15.124 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_crash", 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.125 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.126 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.126 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share metadata about Ceph daemon crashes (version, stack straces, etc)", 2023-07-14T11:29:15.126 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.126 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.126 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.127 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.127 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_device": { 2023-07-14T11:29:15.127 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_device", 2023-07-14T11:29:15.127 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.127 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.128 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.128 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "True", 2023-07-14T11:29:15.128 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.128 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.128 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.129 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)", 2023-07-14T11:29:15.129 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.129 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.129 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.129 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.130 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_ident": { 2023-07-14T11:29:15.130 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_ident", 2023-07-14T11:29:15.130 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.130 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.130 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.131 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.131 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.131 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.131 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.131 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share a user-provided description and/or contact email for the cluster", 2023-07-14T11:29:15.132 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.132 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.132 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.132 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.132 INFO:teuthology.orchestra.run.smithi040.stdout: "channel_perf": { 2023-07-14T11:29:15.133 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "channel_perf", 2023-07-14T11:29:15.133 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.133 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.133 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.133 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Share various performance metrics of a cluster", 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.134 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.135 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.135 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.135 INFO:teuthology.orchestra.run.smithi040.stdout: "contact": { 2023-07-14T11:29:15.135 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "contact", 2023-07-14T11:29:15.135 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.136 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.136 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.136 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.136 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.136 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.137 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.137 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.137 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.137 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.137 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.138 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.138 INFO:teuthology.orchestra.run.smithi040.stdout: "description": { 2023-07-14T11:29:15.138 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "description", 2023-07-14T11:29:15.138 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.138 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.139 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.140 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.140 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.140 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.140 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.140 INFO:teuthology.orchestra.run.smithi040.stdout: "device_url": { 2023-07-14T11:29:15.141 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "device_url", 2023-07-14T11:29:15.141 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.141 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.141 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.141 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/device", 2023-07-14T11:29:15.142 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.142 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.142 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.142 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.142 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.143 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.143 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.143 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.143 INFO:teuthology.orchestra.run.smithi040.stdout: "enabled": { 2023-07-14T11:29:15.143 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "enabled", 2023-07-14T11:29:15.144 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.144 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.144 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.144 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.145 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.146 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.146 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.146 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:15.146 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:15.146 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.147 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.147 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.147 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "24", 2023-07-14T11:29:15.147 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "8", 2023-07-14T11:29:15.147 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.148 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.148 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.148 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.148 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.148 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.149 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.149 INFO:teuthology.orchestra.run.smithi040.stdout: "last_opt_revision": { 2023-07-14T11:29:15.149 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "last_opt_revision", 2023-07-14T11:29:15.149 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.149 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "1", 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.150 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.151 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.151 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.151 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.151 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.151 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard": { 2023-07-14T11:29:15.152 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard", 2023-07-14T11:29:15.152 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.152 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.152 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.153 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.153 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.153 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.153 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.153 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.154 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.154 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.154 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.154 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.154 INFO:teuthology.orchestra.run.smithi040.stdout: "leaderboard_description": { 2023-07-14T11:29:15.155 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "leaderboard_description", 2023-07-14T11:29:15.155 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.155 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.155 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.155 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.156 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.156 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.156 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.156 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.156 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.157 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.157 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.157 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.157 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.157 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.158 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.158 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.158 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.158 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.158 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.159 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.159 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.159 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.159 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.159 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.160 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.160 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.160 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.160 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.160 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.161 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.161 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.161 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.161 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.161 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.162 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.163 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.163 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.163 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.163 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.163 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.164 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.164 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.164 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.164 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.164 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.165 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.165 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.165 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.165 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.165 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.166 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.166 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.166 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.166 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.166 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.167 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.167 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.167 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.167 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.167 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.168 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.168 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.168 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.168 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.168 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.169 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.169 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.169 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.169 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.169 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.170 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.170 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.170 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.170 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.170 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: "organization": { 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "organization", 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.171 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.172 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.172 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.172 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.172 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.173 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.173 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.173 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.173 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.173 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.174 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.174 INFO:teuthology.orchestra.run.smithi040.stdout: "proxy": { 2023-07-14T11:29:15.174 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "proxy", 2023-07-14T11:29:15.174 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.174 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.175 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.175 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.175 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.175 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.175 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.176 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.176 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.176 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.176 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.176 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "url": { 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "url", 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.177 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "https://telemetry.ceph.com/report", 2023-07-14T11:29:15.178 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.178 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.178 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.178 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.178 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.179 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.179 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.179 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.179 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.179 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.180 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.180 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "test_orchestrator", 2023-07-14T11:29:15.180 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.180 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.180 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.181 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.181 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.181 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.181 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.181 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.182 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.183 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.183 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.183 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.183 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.183 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.184 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.184 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.184 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.184 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.184 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.185 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.185 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.185 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.185 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.185 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.186 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.186 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.186 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.186 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.186 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.187 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.188 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.188 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.188 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.188 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.188 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.189 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.189 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.189 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.189 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.189 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.190 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.190 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.190 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.190 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.190 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.191 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.191 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.191 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.191 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.191 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.192 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.192 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.192 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.192 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.192 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.193 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.193 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.193 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.193 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.193 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.194 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.194 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.194 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.194 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.194 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.195 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.195 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "volumes", 2023-07-14T11:29:15.195 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.195 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.196 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.196 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.196 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.196 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.197 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.197 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.197 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.197 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.197 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.198 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.198 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.198 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.198 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.199 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.199 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.199 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.199 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.200 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.200 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.200 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.200 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.201 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.201 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.201 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.202 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.202 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.202 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.202 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.203 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.203 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.203 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.203 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.204 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.204 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.204 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.205 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.205 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.205 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.205 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.206 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.206 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.206 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.206 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.207 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.207 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.207 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.208 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.208 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.208 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.208 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.208 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.209 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.209 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.209 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.209 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.209 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.210 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.211 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.211 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.211 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.211 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.211 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.212 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.212 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.212 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.212 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.212 INFO:teuthology.orchestra.run.smithi040.stdout: "max_concurrent_clones": { 2023-07-14T11:29:15.213 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "max_concurrent_clones", 2023-07-14T11:29:15.213 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.213 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.213 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.213 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "4", 2023-07-14T11:29:15.214 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.214 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.214 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.214 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Number of asynchronous cloner threads", 2023-07-14T11:29:15.214 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.215 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.215 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.215 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.215 INFO:teuthology.orchestra.run.smithi040.stdout: "snapshot_clone_delay": { 2023-07-14T11:29:15.215 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snapshot_clone_delay", 2023-07-14T11:29:15.216 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.216 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.216 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.216 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "0", 2023-07-14T11:29:15.216 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.217 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.217 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.217 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "Delay clone begin operation by snapshot_clone_delay seconds", 2023-07-14T11:29:15.217 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.217 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.218 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.218 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.218 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.218 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.218 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix", 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "can_run": true, 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "error_string": "", 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "module_options": { 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "discovery_interval": { 2023-07-14T11:29:15.219 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "discovery_interval", 2023-07-14T11:29:15.220 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "uint", 2023-07-14T11:29:15.220 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.220 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.220 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "100", 2023-07-14T11:29:15.220 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.221 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.221 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.221 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.221 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.221 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.222 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.222 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.222 INFO:teuthology.orchestra.run.smithi040.stdout: "identifier": { 2023-07-14T11:29:15.222 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "identifier", 2023-07-14T11:29:15.222 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.223 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.224 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.224 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.224 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.224 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.225 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.225 INFO:teuthology.orchestra.run.smithi040.stdout: "interval": { 2023-07-14T11:29:15.225 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "interval", 2023-07-14T11:29:15.225 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "secs", 2023-07-14T11:29:15.225 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.226 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.226 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "60", 2023-07-14T11:29:15.226 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.226 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.226 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.227 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.227 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.227 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.227 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.227 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.228 INFO:teuthology.orchestra.run.smithi040.stdout: "log_level": { 2023-07-14T11:29:15.228 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_level", 2023-07-14T11:29:15.228 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.228 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.228 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.229 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.229 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.229 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.229 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.229 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.230 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.231 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.231 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.231 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.231 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.231 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.232 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster": { 2023-07-14T11:29:15.232 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster", 2023-07-14T11:29:15.232 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.232 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.232 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.233 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.233 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.233 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.233 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.233 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.234 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.234 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.234 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.234 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_cluster_level": { 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_cluster_level", 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.235 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "info", 2023-07-14T11:29:15.236 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.236 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.236 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [ 2023-07-14T11:29:15.236 INFO:teuthology.orchestra.run.smithi040.stdout: "", 2023-07-14T11:29:15.236 INFO:teuthology.orchestra.run.smithi040.stdout: "critical", 2023-07-14T11:29:15.237 INFO:teuthology.orchestra.run.smithi040.stdout: "debug", 2023-07-14T11:29:15.237 INFO:teuthology.orchestra.run.smithi040.stdout: "error", 2023-07-14T11:29:15.237 INFO:teuthology.orchestra.run.smithi040.stdout: "info", 2023-07-14T11:29:15.237 INFO:teuthology.orchestra.run.smithi040.stdout: "warning" 2023-07-14T11:29:15.237 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.238 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.238 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.238 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.238 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.238 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.239 INFO:teuthology.orchestra.run.smithi040.stdout: "log_to_file": { 2023-07-14T11:29:15.239 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "log_to_file", 2023-07-14T11:29:15.239 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "bool", 2023-07-14T11:29:15.239 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.239 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 1, 2023-07-14T11:29:15.240 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "False", 2023-07-14T11:29:15.240 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.240 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.240 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.240 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.241 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.241 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.241 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.241 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.241 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_host": { 2023-07-14T11:29:15.242 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_host", 2023-07-14T11:29:15.242 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.242 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.242 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.242 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "", 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.243 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.244 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.244 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.244 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_port": { 2023-07-14T11:29:15.244 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_port", 2023-07-14T11:29:15.245 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "int", 2023-07-14T11:29:15.245 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.245 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.245 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "10051", 2023-07-14T11:29:15.245 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.246 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.247 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.247 INFO:teuthology.orchestra.run.smithi040.stdout: "zabbix_sender": { 2023-07-14T11:29:15.247 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "zabbix_sender", 2023-07-14T11:29:15.247 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "str", 2023-07-14T11:29:15.247 INFO:teuthology.orchestra.run.smithi040.stdout: "level": "advanced", 2023-07-14T11:29:15.248 INFO:teuthology.orchestra.run.smithi040.stdout: "flags": 0, 2023-07-14T11:29:15.248 INFO:teuthology.orchestra.run.smithi040.stdout: "default_value": "/usr/bin/zabbix_sender", 2023-07-14T11:29:15.248 INFO:teuthology.orchestra.run.smithi040.stdout: "min": "", 2023-07-14T11:29:15.248 INFO:teuthology.orchestra.run.smithi040.stdout: "max": "", 2023-07-14T11:29:15.248 INFO:teuthology.orchestra.run.smithi040.stdout: "enum_allowed": [], 2023-07-14T11:29:15.249 INFO:teuthology.orchestra.run.smithi040.stdout: "desc": "", 2023-07-14T11:29:15.249 INFO:teuthology.orchestra.run.smithi040.stdout: "long_desc": "", 2023-07-14T11:29:15.249 INFO:teuthology.orchestra.run.smithi040.stdout: "tags": [], 2023-07-14T11:29:15.249 INFO:teuthology.orchestra.run.smithi040.stdout: "see_also": [] 2023-07-14T11:29:15.249 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: "services": { 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: "dashboard": "https://172.21.15.40:8443/" 2023-07-14T11:29:15.250 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.251 INFO:teuthology.orchestra.run.smithi040.stdout: "always_on_modules": { 2023-07-14T11:29:15.251 INFO:teuthology.orchestra.run.smithi040.stdout: "octopus": [ 2023-07-14T11:29:15.251 INFO:teuthology.orchestra.run.smithi040.stdout: "balancer", 2023-07-14T11:29:15.251 INFO:teuthology.orchestra.run.smithi040.stdout: "crash", 2023-07-14T11:29:15.251 INFO:teuthology.orchestra.run.smithi040.stdout: "devicehealth", 2023-07-14T11:29:15.252 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator", 2023-07-14T11:29:15.252 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_autoscaler", 2023-07-14T11:29:15.252 INFO:teuthology.orchestra.run.smithi040.stdout: "progress", 2023-07-14T11:29:15.252 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_support", 2023-07-14T11:29:15.252 INFO:teuthology.orchestra.run.smithi040.stdout: "status", 2023-07-14T11:29:15.253 INFO:teuthology.orchestra.run.smithi040.stdout: "telemetry", 2023-07-14T11:29:15.253 INFO:teuthology.orchestra.run.smithi040.stdout: "volumes" 2023-07-14T11:29:15.253 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.253 INFO:teuthology.orchestra.run.smithi040.stdout: "pacific": [ 2023-07-14T11:29:15.253 INFO:teuthology.orchestra.run.smithi040.stdout: "balancer", 2023-07-14T11:29:15.254 INFO:teuthology.orchestra.run.smithi040.stdout: "crash", 2023-07-14T11:29:15.254 INFO:teuthology.orchestra.run.smithi040.stdout: "devicehealth", 2023-07-14T11:29:15.254 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator", 2023-07-14T11:29:15.254 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_autoscaler", 2023-07-14T11:29:15.254 INFO:teuthology.orchestra.run.smithi040.stdout: "progress", 2023-07-14T11:29:15.255 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_support", 2023-07-14T11:29:15.255 INFO:teuthology.orchestra.run.smithi040.stdout: "status", 2023-07-14T11:29:15.255 INFO:teuthology.orchestra.run.smithi040.stdout: "telemetry", 2023-07-14T11:29:15.255 INFO:teuthology.orchestra.run.smithi040.stdout: "volumes" 2023-07-14T11:29:15.255 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.256 INFO:teuthology.orchestra.run.smithi040.stdout: "quincy": [ 2023-07-14T11:29:15.256 INFO:teuthology.orchestra.run.smithi040.stdout: "balancer", 2023-07-14T11:29:15.256 INFO:teuthology.orchestra.run.smithi040.stdout: "crash", 2023-07-14T11:29:15.256 INFO:teuthology.orchestra.run.smithi040.stdout: "devicehealth", 2023-07-14T11:29:15.256 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator", 2023-07-14T11:29:15.257 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_autoscaler", 2023-07-14T11:29:15.257 INFO:teuthology.orchestra.run.smithi040.stdout: "progress", 2023-07-14T11:29:15.257 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_support", 2023-07-14T11:29:15.257 INFO:teuthology.orchestra.run.smithi040.stdout: "status", 2023-07-14T11:29:15.257 INFO:teuthology.orchestra.run.smithi040.stdout: "telemetry", 2023-07-14T11:29:15.258 INFO:teuthology.orchestra.run.smithi040.stdout: "volumes" 2023-07-14T11:29:15.258 INFO:teuthology.orchestra.run.smithi040.stdout: ], 2023-07-14T11:29:15.258 INFO:teuthology.orchestra.run.smithi040.stdout: "reef": [ 2023-07-14T11:29:15.258 INFO:teuthology.orchestra.run.smithi040.stdout: "balancer", 2023-07-14T11:29:15.258 INFO:teuthology.orchestra.run.smithi040.stdout: "crash", 2023-07-14T11:29:15.259 INFO:teuthology.orchestra.run.smithi040.stdout: "devicehealth", 2023-07-14T11:29:15.259 INFO:teuthology.orchestra.run.smithi040.stdout: "orchestrator", 2023-07-14T11:29:15.259 INFO:teuthology.orchestra.run.smithi040.stdout: "pg_autoscaler", 2023-07-14T11:29:15.259 INFO:teuthology.orchestra.run.smithi040.stdout: "progress", 2023-07-14T11:29:15.259 INFO:teuthology.orchestra.run.smithi040.stdout: "rbd_support", 2023-07-14T11:29:15.260 INFO:teuthology.orchestra.run.smithi040.stdout: "status", 2023-07-14T11:29:15.260 INFO:teuthology.orchestra.run.smithi040.stdout: "telemetry", 2023-07-14T11:29:15.260 INFO:teuthology.orchestra.run.smithi040.stdout: "volumes" 2023-07-14T11:29:15.260 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.260 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.261 INFO:teuthology.orchestra.run.smithi040.stdout: "last_failure_osd_epoch": 93, 2023-07-14T11:29:15.261 INFO:teuthology.orchestra.run.smithi040.stdout: "active_clients": [ 2023-07-14T11:29:15.261 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.261 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "devicehealth", 2023-07-14T11:29:15.261 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.262 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.262 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.262 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.262 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 2696552444 2023-07-14T11:29:15.262 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.263 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.263 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.263 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.263 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "libcephsqlite", 2023-07-14T11:29:15.263 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 314680040 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.264 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.265 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.265 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.265 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "rbd_support", 2023-07-14T11:29:15.265 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.265 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.266 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.266 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.266 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 2175378788 2023-07-14T11:29:15.266 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.266 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.267 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.267 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.267 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snap_schedule", 2023-07-14T11:29:15.267 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.267 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 2077295952 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.268 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.269 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.269 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "snap_schedule", 2023-07-14T11:29:15.269 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.269 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.269 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.270 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.270 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 3143451739 2023-07-14T11:29:15.270 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.270 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.270 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:15.271 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.271 INFO:teuthology.orchestra.run.smithi040.stdout: "name": "volumes", 2023-07-14T11:29:15.271 INFO:teuthology.orchestra.run.smithi040.stdout: "addrvec": [ 2023-07-14T11:29:15.271 INFO:teuthology.orchestra.run.smithi040.stdout: { 2023-07-14T11:29:15.271 INFO:teuthology.orchestra.run.smithi040.stdout: "type": "v2", 2023-07-14T11:29:15.272 INFO:teuthology.orchestra.run.smithi040.stdout: "addr": "172.21.15.40:0", 2023-07-14T11:29:15.272 INFO:teuthology.orchestra.run.smithi040.stdout: "nonce": 1943739353 2023-07-14T11:29:15.272 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.272 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.272 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:15.273 INFO:teuthology.orchestra.run.smithi040.stdout: ] 2023-07-14T11:29:15.273 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:15.274 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:11 smithi045 ceph-mon[128791]: pgmap v535: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:15.275 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1099852310' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:29:15.275 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:11 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3217748928' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2023-07-14T11:29:15.275 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:13 smithi045 ceph-mon[128791]: pgmap v536: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:15.275 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:11 smithi130 ceph-mon[129405]: pgmap v535: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:15.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1099852310' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:29:15.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:11 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3217748928' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2023-07-14T11:29:15.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:13 smithi130 ceph-mon[129405]: pgmap v536: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:15.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 msgr2=0x7f88f40a37b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:15.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 0x7f88f40a37b0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f89041993a0 tx=0x7f88f0009260 comp rx=0 tx=0).stop 2023-07-14T11:29:15.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 msgr2=0x7f8904197d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:15.277 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 crypto rx=0x7f88f8005720 tx=0x7f88f8005750 comp rx=0 tx=0).stop 2023-07-14T11:29:15.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 shutdown_connections 2023-07-14T11:29:15.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f88f40a1300 0x7f88f40a37b0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:15.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8904105b00 0x7f890419c730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:15.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f89040ffaf0 0x7f8904197d90 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:15.278 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 --2- 172.21.15.40:0/3217748928 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89041004a0 0x7f89041982d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:15.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.685+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 >> 172.21.15.40:0/3217748928 conn(0x7f89040fb630 msgr2=0x7f890410ad90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:15.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.686+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 shutdown_connections 2023-07-14T11:29:15.279 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:11.686+0000 7f890abff700 1 -- 172.21.15.40:0/3217748928 wait complete. 2023-07-14T11:29:15.286 DEBUG:tasks.check_counter:Getting stats from a 2023-07-14T11:29:15.287 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.a perf dump 2023-07-14T11:29:15.288 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:11 smithi040 ceph-mon[97954]: pgmap v535: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:15.288 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1099852310' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:29:15.288 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:11 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3217748928' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json-pretty"}]: dispatch 2023-07-14T11:29:15.288 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:13 smithi040 ceph-mon[97954]: pgmap v536: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 16 KiB/s rd, 53 KiB/s wr, 27 op/s 2023-07-14T11:29:16.027 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:29:16.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:15 smithi130 ceph-mon[129405]: osdmap e115: 12 total, 12 up, 12 in 2023-07-14T11:29:16.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:15 smithi130 ceph-mon[129405]: pgmap v538: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:15 smithi040 ceph-mon[97954]: osdmap e115: 12 total, 12 up, 12 in 2023-07-14T11:29:16.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:15 smithi040 ceph-mon[97954]: pgmap v538: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:15 smithi045 ceph-mon[128791]: osdmap e115: 12 total, 12 up, 12 in 2023-07-14T11:29:16.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:15 smithi045 ceph-mon[128791]: pgmap v538: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:18.037 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:18.037 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:18.038 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 9556, 2023-07-14T11:29:18.038 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 194628, 2023-07-14T11:29:18.038 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 3937093, 2023-07-14T11:29:18.038 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 38659128, 2023-07-14T11:29:18.039 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 17, 2023-07-14T11:29:18.039 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:18.039 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 7.207742287, 2023-07-14T11:29:18.039 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 3.363039601, 2023-07-14T11:29:18.039 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 2.281408426, 2023-07-14T11:29:18.040 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.772603325, 2023-07-14T11:29:18.040 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:18.040 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 194628, 2023-07-14T11:29:18.040 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 6.419837720, 2023-07-14T11:29:18.041 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000032985 2023-07-14T11:29:18.041 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.041 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:18.041 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 199104, 2023-07-14T11:29:18.041 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.049477516, 2023-07-14T11:29:18.042 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000248 2023-07-14T11:29:18.042 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.042 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:18.043 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 787, 2023-07-14T11:29:18.044 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 556, 2023-07-14T11:29:18.044 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 2868189, 2023-07-14T11:29:18.044 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 537207, 2023-07-14T11:29:18.045 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 20, 2023-07-14T11:29:18.045 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:18.045 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.393779008, 2023-07-14T11:29:18.045 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.028733468, 2023-07-14T11:29:18.046 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.075581167, 2023-07-14T11:29:18.046 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.304223253, 2023-07-14T11:29:18.046 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:18.046 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 552, 2023-07-14T11:29:18.047 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.062684236, 2023-07-14T11:29:18.047 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000113558 2023-07-14T11:29:18.048 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.048 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:18.048 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 22, 2023-07-14T11:29:18.048 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000003611, 2023-07-14T11:29:18.049 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000164 2023-07-14T11:29:18.049 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.049 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 2771312, 2023-07-14T11:29:18.049 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 419456 2023-07-14T11:29:18.050 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.050 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:18.050 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 17484, 2023-07-14T11:29:18.050 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 24027, 2023-07-14T11:29:18.050 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 6229372, 2023-07-14T11:29:18.051 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 8831735, 2023-07-14T11:29:18.051 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 10, 2023-07-14T11:29:18.051 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 7, 2023-07-14T11:29:18.051 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 1.544208630, 2023-07-14T11:29:18.052 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.735792894, 2023-07-14T11:29:18.052 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.986053934, 2023-07-14T11:29:18.052 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.003439014, 2023-07-14T11:29:18.052 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:18.052 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 24026, 2023-07-14T11:29:18.053 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.885794556, 2023-07-14T11:29:18.053 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000036868 2023-07-14T11:29:18.053 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.054 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:18.054 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 39459, 2023-07-14T11:29:18.054 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.008328947, 2023-07-14T11:29:18.054 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000211 2023-07-14T11:29:18.054 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.055 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 5456, 2023-07-14T11:29:18.055 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 582736 2023-07-14T11:29:18.055 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.056 INFO:teuthology.orchestra.run.smithi040.stdout: "cct": { 2023-07-14T11:29:18.056 INFO:teuthology.orchestra.run.smithi040.stdout: "total_workers": 1, 2023-07-14T11:29:18.056 INFO:teuthology.orchestra.run.smithi040.stdout: "unhealthy_workers": 0 2023-07-14T11:29:18.056 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.057 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-MDSRank": { 2023-07-14T11:29:18.057 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:18.057 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:18.058 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 2058, 2023-07-14T11:29:18.058 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 37.911670905, 2023-07-14T11:29:18.059 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.018421608 2023-07-14T11:29:18.059 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.059 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.060 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:18.060 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:18.061 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:18.061 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 83, 2023-07-14T11:29:18.062 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.003005445, 2023-07-14T11:29:18.062 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000036210 2023-07-14T11:29:18.063 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.063 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.063 INFO:teuthology.orchestra.run.smithi040.stdout: "mds": { 2023-07-14T11:29:18.064 INFO:teuthology.orchestra.run.smithi040.stdout: "request": 3927, 2023-07-14T11:29:18.064 INFO:teuthology.orchestra.run.smithi040.stdout: "reply": 3902, 2023-07-14T11:29:18.064 INFO:teuthology.orchestra.run.smithi040.stdout: "reply_latency": { 2023-07-14T11:29:18.065 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 3902, 2023-07-14T11:29:18.065 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 1031825.095355806, 2023-07-14T11:29:18.065 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 264.434929614 2023-07-14T11:29:18.065 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.066 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_reply": 1961, 2023-07-14T11:29:18.066 INFO:teuthology.orchestra.run.smithi040.stdout: "forward": 25, 2023-07-14T11:29:18.066 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_complete": 10, 2023-07-14T11:29:18.066 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:18.066 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_commit": 20, 2023-07-14T11:29:18.067 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_split": 1, 2023-07-14T11:29:18.067 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_merge": 0, 2023-07-14T11:29:18.067 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes": 490, 2023-07-14T11:29:18.067 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_top": 98, 2023-07-14T11:29:18.067 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_bottom": 296, 2023-07-14T11:29:18.068 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pin_tail": 96, 2023-07-14T11:29:18.068 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pinned": 350, 2023-07-14T11:29:18.068 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_expired": 204, 2023-07-14T11:29:18.068 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_with_caps": 42, 2023-07-14T11:29:18.068 INFO:teuthology.orchestra.run.smithi040.stdout: "caps": 42, 2023-07-14T11:29:18.069 INFO:teuthology.orchestra.run.smithi040.stdout: "subtrees": 16, 2023-07-14T11:29:18.069 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse": 8181326, 2023-07-14T11:29:18.069 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_hit": 5779372, 2023-07-14T11:29:18.069 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_forward": 13, 2023-07-14T11:29:18.069 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_discover": 40, 2023-07-14T11:29:18.070 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_dir_fetch": 2, 2023-07-14T11:29:18.070 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:18.070 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_lock": 2352996, 2023-07-14T11:29:18.070 INFO:teuthology.orchestra.run.smithi040.stdout: "load_cent": 1000, 2023-07-14T11:29:18.070 INFO:teuthology.orchestra.run.smithi040.stdout: "q": 1, 2023-07-14T11:29:18.071 INFO:teuthology.orchestra.run.smithi040.stdout: "exported": 5, 2023-07-14T11:29:18.071 INFO:teuthology.orchestra.run.smithi040.stdout: "exported_inodes": 107, 2023-07-14T11:29:18.071 INFO:teuthology.orchestra.run.smithi040.stdout: "imported": 7, 2023-07-14T11:29:18.071 INFO:teuthology.orchestra.run.smithi040.stdout: "imported_inodes": 259, 2023-07-14T11:29:18.071 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:18.072 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_backtrace_fetch": 123, 2023-07-14T11:29:18.072 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:18.072 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rfiles": 1028, 2023-07-14T11:29:18.072 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rbytes": 5852694522, 2023-07-14T11:29:18.072 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rsnaps": 4, 2023-07-14T11:29:18.073 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_fetch": 123, 2023-07-14T11:29:18.073 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:18.073 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:18.073 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:18.073 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_inodes": 2, 2023-07-14T11:29:18.074 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:18.074 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dirfrag_rstats": 2, 2023-07-14T11:29:18.074 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_file_inodes": 121, 2023-07-14T11:29:18.074 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_inode_file_caps": 22, 2023-07-14T11:29:18.074 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_revoke": 273, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_grant": 637, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps": 854, 2023-07-14T11:29:18.075 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps_dirty": 459, 2023-07-14T11:29:18.076 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_cap_release": 53, 2023-07-14T11:29:18.076 INFO:teuthology.orchestra.run.smithi040.stdout: "process_request_cap_release": 700 2023-07-14T11:29:18.076 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.076 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_cache": { 2023-07-14T11:29:18.077 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update": 0, 2023-07-14T11:29:18.077 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:18.077 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_try_discover": 0, 2023-07-14T11:29:18.077 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_send_discover": 41, 2023-07-14T11:29:18.077 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_handle_discover": 114, 2023-07-14T11:29:18.078 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays": 6, 2023-07-14T11:29:18.078 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:18.078 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:18.078 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_created": 176, 2023-07-14T11:29:18.078 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_enqueued": 8, 2023-07-14T11:29:18.079 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_reintegrated": 2926, 2023-07-14T11:29:18.079 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_migrated": 23, 2023-07-14T11:29:18.079 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:18.079 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:18.079 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:18.080 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_started": 0, 2023-07-14T11:29:18.080 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_completed": 0, 2023-07-14T11:29:18.080 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:18.080 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_exportdir": 5, 2023-07-14T11:29:18.080 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_flush": 0, 2023-07-14T11:29:18.081 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragmentdir": 1, 2023-07-14T11:29:18.081 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:18.081 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_inodestats": 0 2023-07-14T11:29:18.081 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.081 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_log": { 2023-07-14T11:29:18.082 INFO:teuthology.orchestra.run.smithi040.stdout: "evadd": 1549, 2023-07-14T11:29:18.082 INFO:teuthology.orchestra.run.smithi040.stdout: "evex": 0, 2023-07-14T11:29:18.082 INFO:teuthology.orchestra.run.smithi040.stdout: "evtrm": 0, 2023-07-14T11:29:18.082 INFO:teuthology.orchestra.run.smithi040.stdout: "ev": 1549, 2023-07-14T11:29:18.082 INFO:teuthology.orchestra.run.smithi040.stdout: "evexg": 0, 2023-07-14T11:29:18.083 INFO:teuthology.orchestra.run.smithi040.stdout: "evexd": 0, 2023-07-14T11:29:18.083 INFO:teuthology.orchestra.run.smithi040.stdout: "segadd": 2, 2023-07-14T11:29:18.083 INFO:teuthology.orchestra.run.smithi040.stdout: "segex": 0, 2023-07-14T11:29:18.083 INFO:teuthology.orchestra.run.smithi040.stdout: "segtrm": 0, 2023-07-14T11:29:18.083 INFO:teuthology.orchestra.run.smithi040.stdout: "seg": 2, 2023-07-14T11:29:18.084 INFO:teuthology.orchestra.run.smithi040.stdout: "segexg": 0, 2023-07-14T11:29:18.084 INFO:teuthology.orchestra.run.smithi040.stdout: "segexd": 0, 2023-07-14T11:29:18.084 INFO:teuthology.orchestra.run.smithi040.stdout: "expos": 4194304, 2023-07-14T11:29:18.084 INFO:teuthology.orchestra.run.smithi040.stdout: "wrpos": 7582657, 2023-07-14T11:29:18.085 INFO:teuthology.orchestra.run.smithi040.stdout: "rdpos": 0, 2023-07-14T11:29:18.085 INFO:teuthology.orchestra.run.smithi040.stdout: "jlat": { 2023-07-14T11:29:18.085 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1002, 2023-07-14T11:29:18.085 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 7.899951904, 2023-07-14T11:29:18.085 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.007884183 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: "replayed": 0 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_mem": { 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: "ino": 318, 2023-07-14T11:29:18.086 INFO:teuthology.orchestra.run.smithi040.stdout: "ino+": 325, 2023-07-14T11:29:18.087 INFO:teuthology.orchestra.run.smithi040.stdout: "ino-": 7, 2023-07-14T11:29:18.087 INFO:teuthology.orchestra.run.smithi040.stdout: "dir": 34, 2023-07-14T11:29:18.087 INFO:teuthology.orchestra.run.smithi040.stdout: "dir+": 36, 2023-07-14T11:29:18.087 INFO:teuthology.orchestra.run.smithi040.stdout: "dir-": 2, 2023-07-14T11:29:18.087 INFO:teuthology.orchestra.run.smithi040.stdout: "dn": 490, 2023-07-14T11:29:18.088 INFO:teuthology.orchestra.run.smithi040.stdout: "dn+": 735, 2023-07-14T11:29:18.088 INFO:teuthology.orchestra.run.smithi040.stdout: "dn-": 245, 2023-07-14T11:29:18.088 INFO:teuthology.orchestra.run.smithi040.stdout: "cap": 42, 2023-07-14T11:29:18.088 INFO:teuthology.orchestra.run.smithi040.stdout: "cap+": 284, 2023-07-14T11:29:18.088 INFO:teuthology.orchestra.run.smithi040.stdout: "cap-": 242, 2023-07-14T11:29:18.089 INFO:teuthology.orchestra.run.smithi040.stdout: "rss": 90488, 2023-07-14T11:29:18.089 INFO:teuthology.orchestra.run.smithi040.stdout: "heap": 198964 2023-07-14T11:29:18.089 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.089 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_server": { 2023-07-14T11:29:18.089 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_client_request": 5288381, 2023-07-14T11:29:18.090 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_server_request": 3324, 2023-07-14T11:29:18.090 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_request": 3927, 2023-07-14T11:29:18.090 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_session": 351, 2023-07-14T11:29:18.090 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_peer_request": 8174, 2023-07-14T11:29:18.090 INFO:teuthology.orchestra.run.smithi040.stdout: "req_create_latency": { 2023-07-14T11:29:18.091 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 36, 2023-07-14T11:29:18.091 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.069230858, 2023-07-14T11:29:18.091 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.001923079 2023-07-14T11:29:18.091 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.091 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getattr_latency": { 2023-07-14T11:29:18.092 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 30, 2023-07-14T11:29:18.092 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 3.687523457, 2023-07-14T11:29:18.092 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.122917448 2023-07-14T11:29:18.092 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.093 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:18.093 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.093 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.093 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.094 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.094 INFO:teuthology.orchestra.run.smithi040.stdout: "req_link_latency": { 2023-07-14T11:29:18.094 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 163, 2023-07-14T11:29:18.094 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 3.295880416, 2023-07-14T11:29:18.094 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.020220125 2023-07-14T11:29:18.095 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.095 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookup_latency": { 2023-07-14T11:29:18.095 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 434, 2023-07-14T11:29:18.095 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 27.173601607, 2023-07-14T11:29:18.095 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.062611985 2023-07-14T11:29:18.096 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.096 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:18.096 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.096 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.096 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.097 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.097 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupino_latency": { 2023-07-14T11:29:18.097 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.098 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.098 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.099 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.099 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupname_latency": { 2023-07-14T11:29:18.100 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.100 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.101 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.101 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.101 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:18.102 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.102 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.102 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.103 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.103 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:18.104 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.104 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.104 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.104 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.105 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lssnap_latency": { 2023-07-14T11:29:18.105 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.105 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.106 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.106 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.106 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mkdir_latency": { 2023-07-14T11:29:18.106 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.106 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.107 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.107 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.107 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mknod_latency": { 2023-07-14T11:29:18.107 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.107 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.108 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.108 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.108 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mksnap_latency": { 2023-07-14T11:29:18.109 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.109 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.109 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.110 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.111 INFO:teuthology.orchestra.run.smithi040.stdout: "req_open_latency": { 2023-07-14T11:29:18.111 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 84, 2023-07-14T11:29:18.112 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 16.605492805, 2023-07-14T11:29:18.112 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.197684438 2023-07-14T11:29:18.112 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.113 INFO:teuthology.orchestra.run.smithi040.stdout: "req_readdir_latency": { 2023-07-14T11:29:18.113 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 39, 2023-07-14T11:29:18.114 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 10.525063937, 2023-07-14T11:29:18.114 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.269873434 2023-07-14T11:29:18.115 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.115 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rename_latency": { 2023-07-14T11:29:18.115 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 2940, 2023-07-14T11:29:18.115 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 1031763.340293020, 2023-07-14T11:29:18.116 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 350.939911664 2023-07-14T11:29:18.116 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.116 INFO:teuthology.orchestra.run.smithi040.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:18.116 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.117 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.118 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.119 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.119 INFO:teuthology.orchestra.run.smithi040.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:18.119 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.119 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.119 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.120 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.120 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmdir_latency": { 2023-07-14T11:29:18.120 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.120 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.121 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.121 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.121 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:18.121 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.121 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.122 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.122 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.122 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:18.122 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.122 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.123 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.123 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.123 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setattr_latency": { 2023-07-14T11:29:18.123 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.123 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.124 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.124 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.124 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:18.124 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.125 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.125 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.125 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.125 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:18.125 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.126 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.126 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.126 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.126 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setlayout_latency": { 2023-07-14T11:29:18.126 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.127 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.127 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.127 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.127 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setxattr_latency": { 2023-07-14T11:29:18.127 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.128 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.128 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.128 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.128 INFO:teuthology.orchestra.run.smithi040.stdout: "req_symlink_latency": { 2023-07-14T11:29:18.129 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.129 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.129 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.129 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.129 INFO:teuthology.orchestra.run.smithi040.stdout: "req_unlink_latency": { 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 176, 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.398269706, 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.002262896 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:18.130 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:18.131 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:18.131 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.131 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.131 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.131 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.132 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.132 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_sessions": { 2023-07-14T11:29:18.132 INFO:teuthology.orchestra.run.smithi040.stdout: "session_count": 2, 2023-07-14T11:29:18.132 INFO:teuthology.orchestra.run.smithi040.stdout: "session_add": 4, 2023-07-14T11:29:18.132 INFO:teuthology.orchestra.run.smithi040.stdout: "session_remove": 2, 2023-07-14T11:29:18.133 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_open": 2, 2023-07-14T11:29:18.133 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_stale": 0, 2023-07-14T11:29:18.133 INFO:teuthology.orchestra.run.smithi040.stdout: "total_load": 115, 2023-07-14T11:29:18.133 INFO:teuthology.orchestra.run.smithi040.stdout: "average_load": 57, 2023-07-14T11:29:18.133 INFO:teuthology.orchestra.run.smithi040.stdout: "avg_session_uptime": 1001 2023-07-14T11:29:18.134 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.134 INFO:teuthology.orchestra.run.smithi040.stdout: "mempool": { 2023-07-14T11:29:18.134 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:18.134 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:18.134 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:18.135 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:18.135 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:18.135 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:18.135 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:18.135 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:18.136 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:18.137 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:18.137 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:18.137 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:18.137 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:18.137 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:18.138 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:18.138 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:18.138 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:18.138 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:18.138 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:18.139 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:18.139 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:18.139 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:18.139 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:18.139 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:18.140 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:18.140 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:18.140 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_items": 0, 2023-07-14T11:29:18.140 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:18.140 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:18.141 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:18.141 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:18.141 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_bytes": 910676, 2023-07-14T11:29:18.141 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_items": 633, 2023-07-14T11:29:18.141 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:18.142 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:18.142 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_bytes": 0, 2023-07-14T11:29:18.142 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_items": 0, 2023-07-14T11:29:18.142 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:18.142 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:18.143 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:18.143 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:18.143 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:18.143 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_items": 183, 2023-07-14T11:29:18.143 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:18.144 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:18.144 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:18.144 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_items": 0, 2023-07-14T11:29:18.144 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_bytes": 1357283, 2023-07-14T11:29:18.144 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_items": 30949, 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_items": 0, 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_items": 0 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.145 INFO:teuthology.orchestra.run.smithi040.stdout: "objecter": { 2023-07-14T11:29:18.146 INFO:teuthology.orchestra.run.smithi040.stdout: "op_active": 0, 2023-07-14T11:29:18.146 INFO:teuthology.orchestra.run.smithi040.stdout: "op_laggy": 0, 2023-07-14T11:29:18.146 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send": 1303, 2023-07-14T11:29:18.146 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send_bytes": 3608537, 2023-07-14T11:29:18.146 INFO:teuthology.orchestra.run.smithi040.stdout: "op_resend": 27, 2023-07-14T11:29:18.147 INFO:teuthology.orchestra.run.smithi040.stdout: "op_reply": 1303, 2023-07-14T11:29:18.147 INFO:teuthology.orchestra.run.smithi040.stdout: "op_latency": { 2023-07-14T11:29:18.147 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1303, 2023-07-14T11:29:18.147 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 2.998960164, 2023-07-14T11:29:18.147 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.002301581 2023-07-14T11:29:18.148 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.148 INFO:teuthology.orchestra.run.smithi040.stdout: "op_inflight": 0, 2023-07-14T11:29:18.148 INFO:teuthology.orchestra.run.smithi040.stdout: "oplen_avg": { 2023-07-14T11:29:18.148 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1303, 2023-07-14T11:29:18.148 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 1408 2023-07-14T11:29:18.149 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.149 INFO:teuthology.orchestra.run.smithi040.stdout: "op": 1303, 2023-07-14T11:29:18.149 INFO:teuthology.orchestra.run.smithi040.stdout: "op_r": 140, 2023-07-14T11:29:18.149 INFO:teuthology.orchestra.run.smithi040.stdout: "op_w": 1163, 2023-07-14T11:29:18.149 INFO:teuthology.orchestra.run.smithi040.stdout: "op_rmw": 0, 2023-07-14T11:29:18.150 INFO:teuthology.orchestra.run.smithi040.stdout: "op_pg": 0, 2023-07-14T11:29:18.150 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_stat": 0, 2023-07-14T11:29:18.150 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_create": 10, 2023-07-14T11:29:18.150 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_read": 7, 2023-07-14T11:29:18.150 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_write": 1009, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writefull": 32, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writesame": 0, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_append": 0, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_zero": 6, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_truncate": 1, 2023-07-14T11:29:18.151 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_delete": 40, 2023-07-14T11:29:18.152 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_mapext": 0, 2023-07-14T11:29:18.152 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:18.152 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:18.152 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_getxattr": 123, 2023-07-14T11:29:18.152 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_setxattr": 20, 2023-07-14T11:29:18.153 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:18.153 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:18.153 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:18.153 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_call": 0, 2023-07-14T11:29:18.153 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_watch": 0, 2023-07-14T11:29:18.154 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_notify": 0, 2023-07-14T11:29:18.154 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:18.154 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls": 0, 2023-07-14T11:29:18.154 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:18.154 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_other": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_active": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_send": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_resend": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_ping": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_active": 0, 2023-07-14T11:29:18.155 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_send": 0, 2023-07-14T11:29:18.156 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_resend": 0, 2023-07-14T11:29:18.156 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_active": 0, 2023-07-14T11:29:18.156 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_send": 0, 2023-07-14T11:29:18.156 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_resend": 0, 2023-07-14T11:29:18.156 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_active": 0, 2023-07-14T11:29:18.157 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_send": 0, 2023-07-14T11:29:18.157 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_resend": 0, 2023-07-14T11:29:18.157 INFO:teuthology.orchestra.run.smithi040.stdout: "command_active": 0, 2023-07-14T11:29:18.157 INFO:teuthology.orchestra.run.smithi040.stdout: "command_send": 0, 2023-07-14T11:29:18.157 INFO:teuthology.orchestra.run.smithi040.stdout: "command_resend": 0, 2023-07-14T11:29:18.158 INFO:teuthology.orchestra.run.smithi040.stdout: "map_epoch": 115, 2023-07-14T11:29:18.158 INFO:teuthology.orchestra.run.smithi040.stdout: "map_full": 0, 2023-07-14T11:29:18.158 INFO:teuthology.orchestra.run.smithi040.stdout: "map_inc": 25, 2023-07-14T11:29:18.158 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_sessions": 12, 2023-07-14T11:29:18.158 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_open": 12, 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_close": 0, 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_laggy": 0, 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_wr": 109, 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_rd": 20, 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_del": 31 2023-07-14T11:29:18.159 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.160 INFO:teuthology.orchestra.run.smithi040.stdout: "oft": { 2023-07-14T11:29:18.160 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_objs": 1, 2023-07-14T11:29:18.160 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:18.160 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_updates": 0, 2023-07-14T11:29:18.160 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_removes": 0 2023-07-14T11:29:18.161 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.161 INFO:teuthology.orchestra.run.smithi040.stdout: "osd-slow-ops": { 2023-07-14T11:29:18.161 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_ops_count": 4114 2023-07-14T11:29:18.161 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.161 INFO:teuthology.orchestra.run.smithi040.stdout: "purge_queue": { 2023-07-14T11:29:18.162 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:18.162 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops_high_water": 10, 2023-07-14T11:29:18.162 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing": 0, 2023-07-14T11:29:18.162 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_high_water": 2, 2023-07-14T11:29:18.162 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executed": 8, 2023-07-14T11:29:18.163 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:18.163 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.163 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:18.163 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:18.163 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:18.164 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:18.164 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 27823, 2023-07-14T11:29:18.164 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 10491843, 2023-07-14T11:29:18.164 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:18.164 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 27823, 2023-07-14T11:29:18.165 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:18.165 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:18.165 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 27823, 2023-07-14T11:29:18.165 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 10491843, 2023-07-14T11:29:18.166 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:18.166 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.166 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.166 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.166 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:18.167 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:18.168 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:18.168 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:18.168 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:18.168 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 1303, 2023-07-14T11:29:18.168 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 3609249, 2023-07-14T11:29:18.169 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 1303, 2023-07-14T11:29:18.169 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 3609249, 2023-07-14T11:29:18.169 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:18.169 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.169 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.170 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.170 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.170 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.170 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:18.170 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:18.171 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 1024, 2023-07-14T11:29:18.171 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:18.171 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:18.171 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:18.171 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:18.172 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:18.172 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 1303, 2023-07-14T11:29:18.172 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 1303, 2023-07-14T11:29:18.172 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 1303, 2023-07-14T11:29:18.172 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 1303, 2023-07-14T11:29:18.173 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:18.173 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.173 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.173 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.173 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:18.174 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 8, 2023-07-14T11:29:18.175 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 872, 2023-07-14T11:29:18.175 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:18.175 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 8, 2023-07-14T11:29:18.175 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:18.175 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:18.176 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 7, 2023-07-14T11:29:18.176 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 872, 2023-07-14T11:29:18.176 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:18.176 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.176 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.177 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.177 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.177 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:18.177 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle-0x5615995ec000": { 2023-07-14T11:29:18.177 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:18.178 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:18.178 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:18.178 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 1549, 2023-07-14T11:29:18.178 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 3388353, 2023-07-14T11:29:18.178 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 1549, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 1002, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 3388353, 2023-07-14T11:29:18.179 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:18.180 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:18.180 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:18.180 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:18.180 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.180 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:18.181 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:18.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:17 smithi040 ceph-mon[97954]: pgmap v539: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:18.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:17 smithi130 ceph-mon[129405]: pgmap v539: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:18.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:17 smithi045 ceph-mon[128791]: pgmap v539: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.5 KiB/s rd, 255 B/s wr, 12 op/s 2023-07-14T11:29:18.523 INFO:tasks.check_counter:Daemon mds.a mds.exported=5 2023-07-14T11:29:18.523 INFO:tasks.check_counter:Daemon mds.a mds.imported=7 2023-07-14T11:29:18.524 INFO:tasks.check_counter:Daemon mds.a mds.root_rsnaps=4 2023-07-14T11:29:18.524 INFO:tasks.check_counter:Daemon mds.a mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:18.525 INFO:tasks.check_counter:Daemon mds.a mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:18.525 DEBUG:tasks.check_counter:Getting stats from d 2023-07-14T11:29:18.525 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.d perf dump 2023-07-14T11:29:19.266 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:29:20.116 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:19 smithi040 ceph-mon[97954]: pgmap v540: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:29:20.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:19 smithi130 ceph-mon[129405]: pgmap v540: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:29:20.314 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:19 smithi045 ceph-mon[128791]: pgmap v540: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.2 KiB/s rd, 205 B/s wr, 10 op/s 2023-07-14T11:29:20.820 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:20.821 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:20.821 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 2360, 2023-07-14T11:29:20.822 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 2362, 2023-07-14T11:29:20.822 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 3419780, 2023-07-14T11:29:20.822 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 733961, 2023-07-14T11:29:20.822 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 6, 2023-07-14T11:29:20.822 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 2, 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.244406996, 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.086528666, 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.065426516, 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.064638756, 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:20.823 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 2360, 2023-07-14T11:29:20.824 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.112037443, 2023-07-14T11:29:20.824 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000047473 2023-07-14T11:29:20.824 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.824 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:20.825 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.825 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.825 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.826 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.826 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:20.826 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:20.826 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.826 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 1621, 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 1385, 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 2979506, 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 510227, 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:20.827 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:20.828 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.175747259, 2023-07-14T11:29:20.828 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.052940520, 2023-07-14T11:29:20.828 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.083386854, 2023-07-14T11:29:20.828 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.043489554, 2023-07-14T11:29:20.829 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:20.829 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1385, 2023-07-14T11:29:20.829 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.064544636, 2023-07-14T11:29:20.829 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000046602 2023-07-14T11:29:20.829 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.830 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:20.830 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.830 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.830 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.830 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 2724736, 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 188896 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 4, 2023-07-14T11:29:20.831 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 247, 2023-07-14T11:29:20.832 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 748, 2023-07-14T11:29:20.832 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 651095, 2023-07-14T11:29:20.832 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 8, 2023-07-14T11:29:20.832 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 4, 2023-07-14T11:29:20.832 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.025724863, 2023-07-14T11:29:20.833 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.016556570, 2023-07-14T11:29:20.833 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.005016762, 2023-07-14T11:29:20.833 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.000016339, 2023-07-14T11:29:20.833 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:20.833 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 244, 2023-07-14T11:29:20.834 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.019683120, 2023-07-14T11:29:20.834 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000080668 2023-07-14T11:29:20.834 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.834 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:20.834 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.835 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.835 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.835 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.835 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 480, 2023-07-14T11:29:20.836 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 650784 2023-07-14T11:29:20.836 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.837 INFO:teuthology.orchestra.run.smithi040.stdout: "cct": { 2023-07-14T11:29:20.837 INFO:teuthology.orchestra.run.smithi040.stdout: "total_workers": 1, 2023-07-14T11:29:20.837 INFO:teuthology.orchestra.run.smithi040.stdout: "unhealthy_workers": 0 2023-07-14T11:29:20.838 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.838 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-MDSRank": { 2023-07-14T11:29:20.838 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:20.839 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:20.839 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 5480, 2023-07-14T11:29:20.839 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.160596903, 2023-07-14T11:29:20.839 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000029306 2023-07-14T11:29:20.840 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.840 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.840 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:20.840 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.841 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.842 INFO:teuthology.orchestra.run.smithi040.stdout: "mds": { 2023-07-14T11:29:20.842 INFO:teuthology.orchestra.run.smithi040.stdout: "request": 0, 2023-07-14T11:29:20.842 INFO:teuthology.orchestra.run.smithi040.stdout: "reply": 0, 2023-07-14T11:29:20.842 INFO:teuthology.orchestra.run.smithi040.stdout: "reply_latency": { 2023-07-14T11:29:20.843 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.843 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.843 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.843 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.843 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_reply": 0, 2023-07-14T11:29:20.844 INFO:teuthology.orchestra.run.smithi040.stdout: "forward": 0, 2023-07-14T11:29:20.844 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:20.844 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:20.844 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_commit": 0, 2023-07-14T11:29:20.844 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_split": 0, 2023-07-14T11:29:20.845 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_merge": 0, 2023-07-14T11:29:20.845 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes": 583, 2023-07-14T11:29:20.845 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_top": 0, 2023-07-14T11:29:20.845 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_bottom": 0, 2023-07-14T11:29:20.845 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pin_tail": 583, 2023-07-14T11:29:20.846 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pinned": 583, 2023-07-14T11:29:20.846 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_expired": 7, 2023-07-14T11:29:20.846 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_with_caps": 0, 2023-07-14T11:29:20.846 INFO:teuthology.orchestra.run.smithi040.stdout: "caps": 0, 2023-07-14T11:29:20.846 INFO:teuthology.orchestra.run.smithi040.stdout: "subtrees": 41, 2023-07-14T11:29:20.847 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse": 0, 2023-07-14T11:29:20.847 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_hit": 0, 2023-07-14T11:29:20.847 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_forward": 0, 2023-07-14T11:29:20.847 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_discover": 0, 2023-07-14T11:29:20.847 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:20.848 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:20.848 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_lock": 0, 2023-07-14T11:29:20.848 INFO:teuthology.orchestra.run.smithi040.stdout: "load_cent": 0, 2023-07-14T11:29:20.848 INFO:teuthology.orchestra.run.smithi040.stdout: "q": 0, 2023-07-14T11:29:20.848 INFO:teuthology.orchestra.run.smithi040.stdout: "exported": 0, 2023-07-14T11:29:20.849 INFO:teuthology.orchestra.run.smithi040.stdout: "exported_inodes": 0, 2023-07-14T11:29:20.849 INFO:teuthology.orchestra.run.smithi040.stdout: "imported": 0, 2023-07-14T11:29:20.849 INFO:teuthology.orchestra.run.smithi040.stdout: "imported_inodes": 0, 2023-07-14T11:29:20.849 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:20.849 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_backtrace_fetch": 0, 2023-07-14T11:29:20.850 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:20.850 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rfiles": 0, 2023-07-14T11:29:20.850 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rbytes": 0, 2023-07-14T11:29:20.850 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rsnaps": 0, 2023-07-14T11:29:20.850 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_fetch": 0, 2023-07-14T11:29:20.851 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:20.851 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:20.851 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:20.851 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:20.851 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:20.852 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:20.852 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_file_inodes": 0, 2023-07-14T11:29:20.852 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_inode_file_caps": 0, 2023-07-14T11:29:20.852 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_revoke": 0, 2023-07-14T11:29:20.852 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_grant": 0, 2023-07-14T11:29:20.853 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:20.853 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:20.853 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:20.853 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps": 0, 2023-07-14T11:29:20.853 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps_dirty": 0, 2023-07-14T11:29:20.854 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_cap_release": 0, 2023-07-14T11:29:20.854 INFO:teuthology.orchestra.run.smithi040.stdout: "process_request_cap_release": 0 2023-07-14T11:29:20.854 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.854 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_cache": { 2023-07-14T11:29:20.854 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update": 0, 2023-07-14T11:29:20.855 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:20.855 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_try_discover": 0, 2023-07-14T11:29:20.855 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_send_discover": 0, 2023-07-14T11:29:20.855 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_handle_discover": 0, 2023-07-14T11:29:20.855 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays": 0, 2023-07-14T11:29:20.856 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:20.856 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:20.856 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_created": 0, 2023-07-14T11:29:20.856 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_enqueued": 0, 2023-07-14T11:29:20.856 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_reintegrated": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_migrated": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_started": 0, 2023-07-14T11:29:20.857 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_completed": 0, 2023-07-14T11:29:20.858 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:20.858 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_exportdir": 0, 2023-07-14T11:29:20.858 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_flush": 0, 2023-07-14T11:29:20.858 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragmentdir": 0, 2023-07-14T11:29:20.858 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:20.859 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_inodestats": 0 2023-07-14T11:29:20.859 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.859 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_log": { 2023-07-14T11:29:20.859 INFO:teuthology.orchestra.run.smithi040.stdout: "evadd": 0, 2023-07-14T11:29:20.859 INFO:teuthology.orchestra.run.smithi040.stdout: "evex": 0, 2023-07-14T11:29:20.860 INFO:teuthology.orchestra.run.smithi040.stdout: "evtrm": 0, 2023-07-14T11:29:20.860 INFO:teuthology.orchestra.run.smithi040.stdout: "ev": 0, 2023-07-14T11:29:20.860 INFO:teuthology.orchestra.run.smithi040.stdout: "evexg": 0, 2023-07-14T11:29:20.860 INFO:teuthology.orchestra.run.smithi040.stdout: "evexd": 0, 2023-07-14T11:29:20.860 INFO:teuthology.orchestra.run.smithi040.stdout: "segadd": 0, 2023-07-14T11:29:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: "segex": 0, 2023-07-14T11:29:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: "segtrm": 0, 2023-07-14T11:29:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: "seg": 2, 2023-07-14T11:29:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: "segexg": 0, 2023-07-14T11:29:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: "segexd": 0, 2023-07-14T11:29:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: "expos": 4194304, 2023-07-14T11:29:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: "wrpos": 0, 2023-07-14T11:29:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: "rdpos": 6911863, 2023-07-14T11:29:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: "jlat": { 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: "replayed": 1746 2023-07-14T11:29:20.863 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.864 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_mem": { 2023-07-14T11:29:20.864 INFO:teuthology.orchestra.run.smithi040.stdout: "ino": 570, 2023-07-14T11:29:20.864 INFO:teuthology.orchestra.run.smithi040.stdout: "ino+": 833, 2023-07-14T11:29:20.864 INFO:teuthology.orchestra.run.smithi040.stdout: "ino-": 263, 2023-07-14T11:29:20.864 INFO:teuthology.orchestra.run.smithi040.stdout: "dir": 44, 2023-07-14T11:29:20.865 INFO:teuthology.orchestra.run.smithi040.stdout: "dir+": 61, 2023-07-14T11:29:20.865 INFO:teuthology.orchestra.run.smithi040.stdout: "dir-": 17, 2023-07-14T11:29:20.865 INFO:teuthology.orchestra.run.smithi040.stdout: "dn": 583, 2023-07-14T11:29:20.865 INFO:teuthology.orchestra.run.smithi040.stdout: "dn+": 853, 2023-07-14T11:29:20.865 INFO:teuthology.orchestra.run.smithi040.stdout: "dn-": 270, 2023-07-14T11:29:20.866 INFO:teuthology.orchestra.run.smithi040.stdout: "cap": 0, 2023-07-14T11:29:20.866 INFO:teuthology.orchestra.run.smithi040.stdout: "cap+": 0, 2023-07-14T11:29:20.866 INFO:teuthology.orchestra.run.smithi040.stdout: "cap-": 0, 2023-07-14T11:29:20.866 INFO:teuthology.orchestra.run.smithi040.stdout: "rss": 51420, 2023-07-14T11:29:20.866 INFO:teuthology.orchestra.run.smithi040.stdout: "heap": 207156 2023-07-14T11:29:20.867 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.867 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_server": { 2023-07-14T11:29:20.867 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_client_request": 0, 2023-07-14T11:29:20.867 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_server_request": 0, 2023-07-14T11:29:20.867 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_request": 0, 2023-07-14T11:29:20.868 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_session": 0, 2023-07-14T11:29:20.868 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_peer_request": 0, 2023-07-14T11:29:20.868 INFO:teuthology.orchestra.run.smithi040.stdout: "req_create_latency": { 2023-07-14T11:29:20.868 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.868 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.869 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.869 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.869 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getattr_latency": { 2023-07-14T11:29:20.869 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.869 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.870 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.870 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.870 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:20.870 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.870 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.871 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.871 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.871 INFO:teuthology.orchestra.run.smithi040.stdout: "req_link_latency": { 2023-07-14T11:29:20.871 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.871 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.872 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.872 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.872 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookup_latency": { 2023-07-14T11:29:20.872 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.872 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.873 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.873 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.873 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:20.873 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.873 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.874 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.874 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.874 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupino_latency": { 2023-07-14T11:29:20.874 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.875 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.875 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.875 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.875 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupname_latency": { 2023-07-14T11:29:20.876 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.876 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.876 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.876 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.876 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:20.877 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.877 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.877 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.877 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.878 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:20.878 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.878 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.878 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.878 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lssnap_latency": { 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.879 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mkdir_latency": { 2023-07-14T11:29:20.880 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.880 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.880 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.880 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.881 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mknod_latency": { 2023-07-14T11:29:20.881 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.881 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.881 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.881 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.882 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mksnap_latency": { 2023-07-14T11:29:20.882 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.882 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.882 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.882 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.883 INFO:teuthology.orchestra.run.smithi040.stdout: "req_open_latency": { 2023-07-14T11:29:20.883 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.883 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.883 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.883 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.884 INFO:teuthology.orchestra.run.smithi040.stdout: "req_readdir_latency": { 2023-07-14T11:29:20.884 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.884 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.884 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.884 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rename_latency": { 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.885 INFO:teuthology.orchestra.run.smithi040.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:20.886 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.886 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.886 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.886 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.886 INFO:teuthology.orchestra.run.smithi040.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:20.887 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.887 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.887 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.887 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.887 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmdir_latency": { 2023-07-14T11:29:20.888 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.888 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.888 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.888 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.888 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:20.889 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.889 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.889 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.889 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.889 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setattr_latency": { 2023-07-14T11:29:20.890 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.891 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.891 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.891 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.891 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:20.891 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.892 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.892 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.892 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.892 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:20.892 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.893 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.893 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.893 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.893 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setlayout_latency": { 2023-07-14T11:29:20.893 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.894 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.894 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.894 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.894 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setxattr_latency": { 2023-07-14T11:29:20.894 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.895 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.895 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.895 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.895 INFO:teuthology.orchestra.run.smithi040.stdout: "req_symlink_latency": { 2023-07-14T11:29:20.895 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.896 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.896 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.896 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.896 INFO:teuthology.orchestra.run.smithi040.stdout: "req_unlink_latency": { 2023-07-14T11:29:20.896 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.897 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.897 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.897 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.897 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:20.897 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:20.898 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:20.898 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.898 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.898 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.898 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.899 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.899 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_sessions": { 2023-07-14T11:29:20.899 INFO:teuthology.orchestra.run.smithi040.stdout: "session_count": 2, 2023-07-14T11:29:20.899 INFO:teuthology.orchestra.run.smithi040.stdout: "session_add": 3, 2023-07-14T11:29:20.899 INFO:teuthology.orchestra.run.smithi040.stdout: "session_remove": 1, 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_open": 3, 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_stale": 0, 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: "total_load": 0, 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: "average_load": 0, 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: "avg_session_uptime": 3802 2023-07-14T11:29:20.900 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.901 INFO:teuthology.orchestra.run.smithi040.stdout: "mempool": { 2023-07-14T11:29:20.901 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:20.901 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:20.901 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:20.901 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:20.902 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:20.902 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:20.902 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:20.902 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:20.902 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:20.903 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:20.903 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:20.903 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:20.903 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:20.903 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:20.904 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:20.904 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:20.904 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:20.904 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:20.904 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:20.905 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:20.905 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:20.905 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:20.905 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:20.905 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:20.906 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:20.907 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:20.907 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_items": 0, 2023-07-14T11:29:20.907 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:20.907 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:20.907 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:20.908 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:20.908 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_bytes": 213260, 2023-07-14T11:29:20.908 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_items": 650, 2023-07-14T11:29:20.908 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:20.908 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:20.909 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_bytes": 0, 2023-07-14T11:29:20.909 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_items": 0, 2023-07-14T11:29:20.909 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:20.909 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:20.909 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:20.910 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:20.910 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:20.910 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_items": 183, 2023-07-14T11:29:20.910 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:20.910 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:20.911 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:20.911 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_items": 0, 2023-07-14T11:29:20.911 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_bytes": 1797181, 2023-07-14T11:29:20.911 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_items": 18722, 2023-07-14T11:29:20.911 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:20.912 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_items": 0, 2023-07-14T11:29:20.912 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:20.912 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_items": 0 2023-07-14T11:29:20.912 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.912 INFO:teuthology.orchestra.run.smithi040.stdout: "objecter": { 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_active": 0, 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_laggy": 0, 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send": 3394, 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send_bytes": 21, 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_resend": 3, 2023-07-14T11:29:20.913 INFO:teuthology.orchestra.run.smithi040.stdout: "op_reply": 3394, 2023-07-14T11:29:20.914 INFO:teuthology.orchestra.run.smithi040.stdout: "op_latency": { 2023-07-14T11:29:20.914 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 3394, 2023-07-14T11:29:20.914 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 1.913974737, 2023-07-14T11:29:20.914 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000563928 2023-07-14T11:29:20.914 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.915 INFO:teuthology.orchestra.run.smithi040.stdout: "op_inflight": 0, 2023-07-14T11:29:20.915 INFO:teuthology.orchestra.run.smithi040.stdout: "oplen_avg": { 2023-07-14T11:29:20.915 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 3394, 2023-07-14T11:29:20.915 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 3395 2023-07-14T11:29:20.915 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.916 INFO:teuthology.orchestra.run.smithi040.stdout: "op": 3394, 2023-07-14T11:29:20.916 INFO:teuthology.orchestra.run.smithi040.stdout: "op_r": 3394, 2023-07-14T11:29:20.916 INFO:teuthology.orchestra.run.smithi040.stdout: "op_w": 0, 2023-07-14T11:29:20.916 INFO:teuthology.orchestra.run.smithi040.stdout: "op_rmw": 0, 2023-07-14T11:29:20.917 INFO:teuthology.orchestra.run.smithi040.stdout: "op_pg": 0, 2023-07-14T11:29:20.917 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_stat": 2172, 2023-07-14T11:29:20.917 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_create": 0, 2023-07-14T11:29:20.917 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_read": 1220, 2023-07-14T11:29:20.918 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_write": 0, 2023-07-14T11:29:20.918 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writefull": 0, 2023-07-14T11:29:20.918 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writesame": 0, 2023-07-14T11:29:20.918 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_append": 0, 2023-07-14T11:29:20.918 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_zero": 0, 2023-07-14T11:29:20.919 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_truncate": 0, 2023-07-14T11:29:20.919 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_delete": 0, 2023-07-14T11:29:20.919 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_mapext": 0, 2023-07-14T11:29:20.919 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:20.919 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:20.920 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_getxattr": 1, 2023-07-14T11:29:20.920 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_setxattr": 0, 2023-07-14T11:29:20.920 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:20.920 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:20.920 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:20.921 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_call": 0, 2023-07-14T11:29:20.921 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_watch": 0, 2023-07-14T11:29:20.921 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_notify": 0, 2023-07-14T11:29:20.921 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:20.921 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls": 0, 2023-07-14T11:29:20.922 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:20.922 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_other": 0, 2023-07-14T11:29:20.922 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_active": 0, 2023-07-14T11:29:20.922 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_send": 0, 2023-07-14T11:29:20.922 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_resend": 0, 2023-07-14T11:29:20.923 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_ping": 0, 2023-07-14T11:29:20.923 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_active": 0, 2023-07-14T11:29:20.923 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_send": 0, 2023-07-14T11:29:20.923 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_resend": 0, 2023-07-14T11:29:20.923 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_active": 0, 2023-07-14T11:29:20.924 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_send": 0, 2023-07-14T11:29:20.924 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_resend": 0, 2023-07-14T11:29:20.924 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_active": 0, 2023-07-14T11:29:20.924 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_send": 0, 2023-07-14T11:29:20.924 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_resend": 0, 2023-07-14T11:29:20.925 INFO:teuthology.orchestra.run.smithi040.stdout: "command_active": 0, 2023-07-14T11:29:20.925 INFO:teuthology.orchestra.run.smithi040.stdout: "command_send": 0, 2023-07-14T11:29:20.925 INFO:teuthology.orchestra.run.smithi040.stdout: "command_resend": 0, 2023-07-14T11:29:20.925 INFO:teuthology.orchestra.run.smithi040.stdout: "map_epoch": 115, 2023-07-14T11:29:20.925 INFO:teuthology.orchestra.run.smithi040.stdout: "map_full": 0, 2023-07-14T11:29:20.926 INFO:teuthology.orchestra.run.smithi040.stdout: "map_inc": 25, 2023-07-14T11:29:20.926 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_sessions": 5, 2023-07-14T11:29:20.926 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_open": 7, 2023-07-14T11:29:20.926 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_close": 2, 2023-07-14T11:29:20.926 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_laggy": 0, 2023-07-14T11:29:20.927 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_wr": 0, 2023-07-14T11:29:20.927 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_rd": 2, 2023-07-14T11:29:20.927 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_del": 0 2023-07-14T11:29:20.927 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.927 INFO:teuthology.orchestra.run.smithi040.stdout: "oft": { 2023-07-14T11:29:20.928 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_objs": 0, 2023-07-14T11:29:20.928 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:20.928 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_updates": 0, 2023-07-14T11:29:20.928 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_removes": 0 2023-07-14T11:29:20.928 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.929 INFO:teuthology.orchestra.run.smithi040.stdout: "osd-slow-ops": { 2023-07-14T11:29:20.929 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_ops_count": 0 2023-07-14T11:29:20.929 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.929 INFO:teuthology.orchestra.run.smithi040.stdout: "purge_queue": { 2023-07-14T11:29:20.929 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:20.930 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:20.930 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing": 0, 2023-07-14T11:29:20.930 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:20.930 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executed": 0, 2023-07-14T11:29:20.930 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:20.931 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 3981, 2023-07-14T11:29:20.932 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 6009475, 2023-07-14T11:29:20.932 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:20.932 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 3981, 2023-07-14T11:29:20.932 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 3981, 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 6009475, 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.933 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.934 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.934 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.934 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.935 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:20.935 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:20.935 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:20.935 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:20.935 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 3394, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 2718138, 2023-07-14T11:29:20.936 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 3394, 2023-07-14T11:29:20.937 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 2718138, 2023-07-14T11:29:20.937 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:20.937 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.937 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.937 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.938 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.938 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.938 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:20.938 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:20.938 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 1024, 2023-07-14T11:29:20.939 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:20.939 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:20.939 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:20.939 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:20.939 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:20.940 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 3394, 2023-07-14T11:29:20.940 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 3394, 2023-07-14T11:29:20.940 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 3394, 2023-07-14T11:29:20.940 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 3394, 2023-07-14T11:29:20.940 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:20.941 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.941 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.941 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.941 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.941 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.942 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:20.942 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:20.942 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:20.942 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:20.942 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:20.943 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:20.943 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:20.943 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:20.943 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:20.943 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:20.944 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 0, 2023-07-14T11:29:20.944 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 0, 2023-07-14T11:29:20.944 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:20.944 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.944 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.945 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.945 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.945 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:20.945 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle-0x563c212ca500": { 2023-07-14T11:29:20.945 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:20.946 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:20.946 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:20.946 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:20.946 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:20.946 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 0, 2023-07-14T11:29:20.947 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:20.948 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:20.948 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:20.948 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:20.948 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.948 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:20.949 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:21.261 INFO:tasks.check_counter:Daemon mds.d mds.exported=0 2023-07-14T11:29:21.261 INFO:tasks.check_counter:Daemon mds.d mds.imported=0 2023-07-14T11:29:21.262 INFO:tasks.check_counter:Daemon mds.d mds.root_rsnaps=0 2023-07-14T11:29:21.262 INFO:tasks.check_counter:Daemon mds.d mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:21.262 INFO:tasks.check_counter:Daemon mds.d mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:21.262 DEBUG:tasks.check_counter:Getting stats from g 2023-07-14T11:29:21.263 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.g perf dump 2023-07-14T11:29:21.996 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:29:22.262 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:21 smithi040 ceph-mon[97954]: pgmap v541: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:22.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:21 smithi130 ceph-mon[129405]: pgmap v541: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:22.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:21 smithi045 ceph-mon[128791]: pgmap v541: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:23.561 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:23.561 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:23.562 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 1631, 2023-07-14T11:29:23.562 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 1633, 2023-07-14T11:29:23.562 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 4701717, 2023-07-14T11:29:23.563 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 507251, 2023-07-14T11:29:23.563 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 6, 2023-07-14T11:29:23.563 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 2, 2023-07-14T11:29:23.563 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.187158596, 2023-07-14T11:29:23.563 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.062850180, 2023-07-14T11:29:23.564 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.050465938, 2023-07-14T11:29:23.564 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.051662991, 2023-07-14T11:29:23.564 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:23.564 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1631, 2023-07-14T11:29:23.564 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.078546435, 2023-07-14T11:29:23.565 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000048158 2023-07-14T11:29:23.565 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.565 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:23.565 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.565 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.566 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.566 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.566 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:23.567 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:23.567 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.567 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:23.567 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 2179, 2023-07-14T11:29:23.568 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 1936, 2023-07-14T11:29:23.568 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 9100602, 2023-07-14T11:29:23.568 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 680592, 2023-07-14T11:29:23.569 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 4, 2023-07-14T11:29:23.569 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:23.569 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.221798707, 2023-07-14T11:29:23.569 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.074990525, 2023-07-14T11:29:23.569 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.106565435, 2023-07-14T11:29:23.570 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.044992482, 2023-07-14T11:29:23.570 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:23.570 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 1936, 2023-07-14T11:29:23.570 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.092302276, 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000047676 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.571 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.572 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.572 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 2750432, 2023-07-14T11:29:23.572 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 187280 2023-07-14T11:29:23.572 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.573 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:23.573 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 447, 2023-07-14T11:29:23.573 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 687, 2023-07-14T11:29:23.573 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 4307455, 2023-07-14T11:29:23.574 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 788341, 2023-07-14T11:29:23.574 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 7, 2023-07-14T11:29:23.574 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 4, 2023-07-14T11:29:23.574 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.074851585, 2023-07-14T11:29:23.574 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.029978655, 2023-07-14T11:29:23.575 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.024580719, 2023-07-14T11:29:23.575 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.011967933, 2023-07-14T11:29:23.575 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:23.576 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 684, 2023-07-14T11:29:23.576 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.042622583, 2023-07-14T11:29:23.576 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000062313 2023-07-14T11:29:23.576 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.577 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:23.577 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.577 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.577 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.577 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 480, 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 650256 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: "cct": { 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: "total_workers": 1, 2023-07-14T11:29:23.578 INFO:teuthology.orchestra.run.smithi040.stdout: "unhealthy_workers": 0 2023-07-14T11:29:23.579 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.579 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-MDSRank": { 2023-07-14T11:29:23.579 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:23.579 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:23.580 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 6099, 2023-07-14T11:29:23.580 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.176360741, 2023-07-14T11:29:23.580 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000028916 2023-07-14T11:29:23.580 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: "queue_len": 0, 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: "complete_latency": { 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.581 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.582 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.582 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.582 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.582 INFO:teuthology.orchestra.run.smithi040.stdout: "mds": { 2023-07-14T11:29:23.582 INFO:teuthology.orchestra.run.smithi040.stdout: "request": 0, 2023-07-14T11:29:23.583 INFO:teuthology.orchestra.run.smithi040.stdout: "reply": 0, 2023-07-14T11:29:23.583 INFO:teuthology.orchestra.run.smithi040.stdout: "reply_latency": { 2023-07-14T11:29:23.583 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.583 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.583 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.584 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.584 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_reply": 0, 2023-07-14T11:29:23.584 INFO:teuthology.orchestra.run.smithi040.stdout: "forward": 0, 2023-07-14T11:29:23.584 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:23.585 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:23.585 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_commit": 0, 2023-07-14T11:29:23.585 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_split": 0, 2023-07-14T11:29:23.586 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_merge": 0, 2023-07-14T11:29:23.586 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes": 707, 2023-07-14T11:29:23.586 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_top": 39, 2023-07-14T11:29:23.587 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_bottom": 65, 2023-07-14T11:29:23.587 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pin_tail": 603, 2023-07-14T11:29:23.588 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_pinned": 651, 2023-07-14T11:29:23.588 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_expired": 8, 2023-07-14T11:29:23.589 INFO:teuthology.orchestra.run.smithi040.stdout: "inodes_with_caps": 0, 2023-07-14T11:29:23.589 INFO:teuthology.orchestra.run.smithi040.stdout: "caps": 0, 2023-07-14T11:29:23.590 INFO:teuthology.orchestra.run.smithi040.stdout: "subtrees": 128, 2023-07-14T11:29:23.590 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse": 0, 2023-07-14T11:29:23.590 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_hit": 0, 2023-07-14T11:29:23.591 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_forward": 0, 2023-07-14T11:29:23.591 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_discover": 0, 2023-07-14T11:29:23.592 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:23.592 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:23.592 INFO:teuthology.orchestra.run.smithi040.stdout: "traverse_lock": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "load_cent": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "q": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "exported": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "exported_inodes": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "imported": 0, 2023-07-14T11:29:23.593 INFO:teuthology.orchestra.run.smithi040.stdout: "imported_inodes": 0, 2023-07-14T11:29:23.594 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:23.594 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_backtrace_fetch": 0, 2023-07-14T11:29:23.594 INFO:teuthology.orchestra.run.smithi040.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:23.594 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rfiles": 1028, 2023-07-14T11:29:23.595 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rbytes": 5852694910, 2023-07-14T11:29:23.595 INFO:teuthology.orchestra.run.smithi040.stdout: "root_rsnaps": 6, 2023-07-14T11:29:23.595 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_fetch": 0, 2023-07-14T11:29:23.595 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:23.596 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:23.596 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:23.596 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:23.596 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:23.597 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:23.597 INFO:teuthology.orchestra.run.smithi040.stdout: "scrub_file_inodes": 0, 2023-07-14T11:29:23.597 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_inode_file_caps": 0, 2023-07-14T11:29:23.597 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_revoke": 0, 2023-07-14T11:29:23.597 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_grant": 0, 2023-07-14T11:29:23.598 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:23.598 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:23.598 INFO:teuthology.orchestra.run.smithi040.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:23.599 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps": 0, 2023-07-14T11:29:23.599 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_caps_dirty": 0, 2023-07-14T11:29:23.599 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_cap_release": 0, 2023-07-14T11:29:23.600 INFO:teuthology.orchestra.run.smithi040.stdout: "process_request_cap_release": 0 2023-07-14T11:29:23.600 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.601 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_cache": { 2023-07-14T11:29:23.601 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update": 0, 2023-07-14T11:29:23.601 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:23.602 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_try_discover": 0, 2023-07-14T11:29:23.602 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_send_discover": 0, 2023-07-14T11:29:23.603 INFO:teuthology.orchestra.run.smithi040.stdout: "dir_handle_discover": 0, 2023-07-14T11:29:23.603 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays": 0, 2023-07-14T11:29:23.603 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:23.604 INFO:teuthology.orchestra.run.smithi040.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:23.604 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_created": 0, 2023-07-14T11:29:23.605 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_enqueued": 0, 2023-07-14T11:29:23.605 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_reintegrated": 0, 2023-07-14T11:29:23.605 INFO:teuthology.orchestra.run.smithi040.stdout: "strays_migrated": 0, 2023-07-14T11:29:23.606 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:23.606 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:23.606 INFO:teuthology.orchestra.run.smithi040.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:23.606 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_started": 0, 2023-07-14T11:29:23.606 INFO:teuthology.orchestra.run.smithi040.stdout: "recovery_completed": 0, 2023-07-14T11:29:23.607 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:23.607 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_exportdir": 0, 2023-07-14T11:29:23.607 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_flush": 0, 2023-07-14T11:29:23.607 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragmentdir": 0, 2023-07-14T11:29:23.608 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:23.608 INFO:teuthology.orchestra.run.smithi040.stdout: "ireq_inodestats": 0 2023-07-14T11:29:23.608 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.608 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_log": { 2023-07-14T11:29:23.608 INFO:teuthology.orchestra.run.smithi040.stdout: "evadd": 0, 2023-07-14T11:29:23.609 INFO:teuthology.orchestra.run.smithi040.stdout: "evex": 0, 2023-07-14T11:29:23.609 INFO:teuthology.orchestra.run.smithi040.stdout: "evtrm": 0, 2023-07-14T11:29:23.609 INFO:teuthology.orchestra.run.smithi040.stdout: "ev": 0, 2023-07-14T11:29:23.609 INFO:teuthology.orchestra.run.smithi040.stdout: "evexg": 0, 2023-07-14T11:29:23.609 INFO:teuthology.orchestra.run.smithi040.stdout: "evexd": 0, 2023-07-14T11:29:23.610 INFO:teuthology.orchestra.run.smithi040.stdout: "segadd": 0, 2023-07-14T11:29:23.610 INFO:teuthology.orchestra.run.smithi040.stdout: "segex": 0, 2023-07-14T11:29:23.610 INFO:teuthology.orchestra.run.smithi040.stdout: "segtrm": 0, 2023-07-14T11:29:23.610 INFO:teuthology.orchestra.run.smithi040.stdout: "seg": 7, 2023-07-14T11:29:23.610 INFO:teuthology.orchestra.run.smithi040.stdout: "segexg": 0, 2023-07-14T11:29:23.611 INFO:teuthology.orchestra.run.smithi040.stdout: "segexd": 0, 2023-07-14T11:29:23.611 INFO:teuthology.orchestra.run.smithi040.stdout: "expos": 4194304, 2023-07-14T11:29:23.611 INFO:teuthology.orchestra.run.smithi040.stdout: "wrpos": 0, 2023-07-14T11:29:23.611 INFO:teuthology.orchestra.run.smithi040.stdout: "rdpos": 17341481, 2023-07-14T11:29:23.611 INFO:teuthology.orchestra.run.smithi040.stdout: "jlat": { 2023-07-14T11:29:23.612 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.612 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.612 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.612 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.612 INFO:teuthology.orchestra.run.smithi040.stdout: "replayed": 4334 2023-07-14T11:29:23.613 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.613 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_mem": { 2023-07-14T11:29:23.613 INFO:teuthology.orchestra.run.smithi040.stdout: "ino": 575, 2023-07-14T11:29:23.613 INFO:teuthology.orchestra.run.smithi040.stdout: "ino+": 1761, 2023-07-14T11:29:23.613 INFO:teuthology.orchestra.run.smithi040.stdout: "ino-": 1186, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dir": 167, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dir+": 187, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dir-": 20, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dn": 707, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dn+": 1866, 2023-07-14T11:29:23.614 INFO:teuthology.orchestra.run.smithi040.stdout: "dn-": 1159, 2023-07-14T11:29:23.615 INFO:teuthology.orchestra.run.smithi040.stdout: "cap": 0, 2023-07-14T11:29:23.615 INFO:teuthology.orchestra.run.smithi040.stdout: "cap+": 0, 2023-07-14T11:29:23.615 INFO:teuthology.orchestra.run.smithi040.stdout: "cap-": 0, 2023-07-14T11:29:23.615 INFO:teuthology.orchestra.run.smithi040.stdout: "rss": 53724, 2023-07-14T11:29:23.615 INFO:teuthology.orchestra.run.smithi040.stdout: "heap": 207156 2023-07-14T11:29:23.616 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.616 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_server": { 2023-07-14T11:29:23.616 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_client_request": 0, 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "dispatch_server_request": 0, 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_request": 0, 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_client_session": 0, 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "handle_peer_request": 0, 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "req_create_latency": { 2023-07-14T11:29:23.617 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.618 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.618 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.618 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.618 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getattr_latency": { 2023-07-14T11:29:23.618 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.619 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.619 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.619 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.619 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:23.619 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.620 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.620 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.620 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.620 INFO:teuthology.orchestra.run.smithi040.stdout: "req_link_latency": { 2023-07-14T11:29:23.620 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.621 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.621 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.621 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.621 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookup_latency": { 2023-07-14T11:29:23.622 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.622 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.622 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.622 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.622 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupino_latency": { 2023-07-14T11:29:23.623 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.624 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.624 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.624 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.624 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupname_latency": { 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:23.625 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.626 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.626 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.626 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.627 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:23.627 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.627 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.627 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.628 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.628 INFO:teuthology.orchestra.run.smithi040.stdout: "req_lssnap_latency": { 2023-07-14T11:29:23.628 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.629 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.629 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.629 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.630 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mkdir_latency": { 2023-07-14T11:29:23.630 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.630 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.631 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.631 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.631 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mknod_latency": { 2023-07-14T11:29:23.631 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.632 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.632 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.632 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.633 INFO:teuthology.orchestra.run.smithi040.stdout: "req_mksnap_latency": { 2023-07-14T11:29:23.633 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.633 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.634 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.634 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.634 INFO:teuthology.orchestra.run.smithi040.stdout: "req_open_latency": { 2023-07-14T11:29:23.635 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.635 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.635 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.636 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.636 INFO:teuthology.orchestra.run.smithi040.stdout: "req_readdir_latency": { 2023-07-14T11:29:23.636 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.637 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.637 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.637 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.638 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rename_latency": { 2023-07-14T11:29:23.638 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.638 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.639 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.639 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.639 INFO:teuthology.orchestra.run.smithi040.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:23.640 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.640 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.640 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.640 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.641 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmdir_latency": { 2023-07-14T11:29:23.642 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.642 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.642 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.642 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.643 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:23.643 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.643 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.643 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.644 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.644 INFO:teuthology.orchestra.run.smithi040.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:23.644 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setattr_latency": { 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.645 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.646 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.646 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.646 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:23.646 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.646 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.647 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.647 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.647 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:23.647 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.647 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.648 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.648 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.648 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setlayout_latency": { 2023-07-14T11:29:23.648 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.648 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.649 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.649 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.649 INFO:teuthology.orchestra.run.smithi040.stdout: "req_setxattr_latency": { 2023-07-14T11:29:23.649 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.649 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.650 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.650 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.650 INFO:teuthology.orchestra.run.smithi040.stdout: "req_symlink_latency": { 2023-07-14T11:29:23.650 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.650 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.651 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.651 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.651 INFO:teuthology.orchestra.run.smithi040.stdout: "req_unlink_latency": { 2023-07-14T11:29:23.651 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.651 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.652 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.652 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.652 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:23.652 INFO:teuthology.orchestra.run.smithi040.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:23.652 INFO:teuthology.orchestra.run.smithi040.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:23.653 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.653 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.653 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.653 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.654 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.654 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_sessions": { 2023-07-14T11:29:23.654 INFO:teuthology.orchestra.run.smithi040.stdout: "session_count": 2, 2023-07-14T11:29:23.654 INFO:teuthology.orchestra.run.smithi040.stdout: "session_add": 4, 2023-07-14T11:29:23.655 INFO:teuthology.orchestra.run.smithi040.stdout: "session_remove": 2, 2023-07-14T11:29:23.655 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_open": 3, 2023-07-14T11:29:23.655 INFO:teuthology.orchestra.run.smithi040.stdout: "sessions_stale": 0, 2023-07-14T11:29:23.655 INFO:teuthology.orchestra.run.smithi040.stdout: "total_load": 0, 2023-07-14T11:29:23.655 INFO:teuthology.orchestra.run.smithi040.stdout: "average_load": 0, 2023-07-14T11:29:23.656 INFO:teuthology.orchestra.run.smithi040.stdout: "avg_session_uptime": 3806 2023-07-14T11:29:23.656 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.656 INFO:teuthology.orchestra.run.smithi040.stdout: "mempool": { 2023-07-14T11:29:23.656 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:23.656 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:23.657 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:23.657 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:23.657 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:23.657 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:23.657 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:23.658 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:23.658 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:23.658 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:23.658 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:23.658 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:23.659 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:23.659 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:23.659 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:23.659 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:23.659 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:23.660 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:23.660 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:23.660 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:23.660 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:23.661 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:23.661 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:23.661 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:23.661 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:23.661 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:23.662 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:23.662 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:23.662 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:23.662 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:23.663 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:23.663 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_items": 0, 2023-07-14T11:29:23.663 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:23.663 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:23.664 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:23.664 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:23.664 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_bytes": 215099, 2023-07-14T11:29:23.664 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_items": 726, 2023-07-14T11:29:23.665 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:23.665 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:23.665 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_bytes": 0, 2023-07-14T11:29:23.665 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_items": 0, 2023-07-14T11:29:23.665 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:23.666 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:23.666 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:23.666 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:23.666 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:23.667 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_items": 183, 2023-07-14T11:29:23.667 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:23.667 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:23.667 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:23.667 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_items": 0, 2023-07-14T11:29:23.668 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_bytes": 2141126, 2023-07-14T11:29:23.668 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_items": 22796, 2023-07-14T11:29:23.668 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:23.669 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_items": 0, 2023-07-14T11:29:23.669 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:23.669 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_items": 0 2023-07-14T11:29:23.669 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.669 INFO:teuthology.orchestra.run.smithi040.stdout: "objecter": { 2023-07-14T11:29:23.670 INFO:teuthology.orchestra.run.smithi040.stdout: "op_active": 0, 2023-07-14T11:29:23.670 INFO:teuthology.orchestra.run.smithi040.stdout: "op_laggy": 0, 2023-07-14T11:29:23.670 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send": 3661, 2023-07-14T11:29:23.670 INFO:teuthology.orchestra.run.smithi040.stdout: "op_send_bytes": 26, 2023-07-14T11:29:23.670 INFO:teuthology.orchestra.run.smithi040.stdout: "op_resend": 4, 2023-07-14T11:29:23.671 INFO:teuthology.orchestra.run.smithi040.stdout: "op_reply": 3661, 2023-07-14T11:29:23.671 INFO:teuthology.orchestra.run.smithi040.stdout: "op_latency": { 2023-07-14T11:29:23.671 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 3661, 2023-07-14T11:29:23.671 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 2.012973381, 2023-07-14T11:29:23.671 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000549842 2023-07-14T11:29:23.672 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.672 INFO:teuthology.orchestra.run.smithi040.stdout: "op_inflight": 0, 2023-07-14T11:29:23.672 INFO:teuthology.orchestra.run.smithi040.stdout: "oplen_avg": { 2023-07-14T11:29:23.672 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 3661, 2023-07-14T11:29:23.673 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 3662 2023-07-14T11:29:23.673 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.673 INFO:teuthology.orchestra.run.smithi040.stdout: "op": 3661, 2023-07-14T11:29:23.673 INFO:teuthology.orchestra.run.smithi040.stdout: "op_r": 3661, 2023-07-14T11:29:23.673 INFO:teuthology.orchestra.run.smithi040.stdout: "op_w": 0, 2023-07-14T11:29:23.674 INFO:teuthology.orchestra.run.smithi040.stdout: "op_rmw": 0, 2023-07-14T11:29:23.674 INFO:teuthology.orchestra.run.smithi040.stdout: "op_pg": 0, 2023-07-14T11:29:23.674 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_stat": 2294, 2023-07-14T11:29:23.674 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_create": 0, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_read": 1364, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_write": 0, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writefull": 0, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_writesame": 0, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_append": 0, 2023-07-14T11:29:23.675 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_zero": 0, 2023-07-14T11:29:23.676 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_truncate": 0, 2023-07-14T11:29:23.676 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_delete": 0, 2023-07-14T11:29:23.676 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_mapext": 0, 2023-07-14T11:29:23.676 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:23.677 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:23.677 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_getxattr": 2, 2023-07-14T11:29:23.677 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_setxattr": 0, 2023-07-14T11:29:23.678 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:23.678 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:23.679 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:23.679 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_call": 0, 2023-07-14T11:29:23.679 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_watch": 0, 2023-07-14T11:29:23.680 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_notify": 0, 2023-07-14T11:29:23.680 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:23.681 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls": 0, 2023-07-14T11:29:23.681 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:23.682 INFO:teuthology.orchestra.run.smithi040.stdout: "osdop_other": 0, 2023-07-14T11:29:23.682 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_active": 0, 2023-07-14T11:29:23.682 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_send": 0, 2023-07-14T11:29:23.683 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_resend": 0, 2023-07-14T11:29:23.683 INFO:teuthology.orchestra.run.smithi040.stdout: "linger_ping": 0, 2023-07-14T11:29:23.684 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_active": 0, 2023-07-14T11:29:23.684 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_send": 0, 2023-07-14T11:29:23.684 INFO:teuthology.orchestra.run.smithi040.stdout: "poolop_resend": 0, 2023-07-14T11:29:23.685 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_active": 0, 2023-07-14T11:29:23.685 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_send": 0, 2023-07-14T11:29:23.685 INFO:teuthology.orchestra.run.smithi040.stdout: "poolstat_resend": 0, 2023-07-14T11:29:23.685 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_active": 0, 2023-07-14T11:29:23.686 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_send": 0, 2023-07-14T11:29:23.686 INFO:teuthology.orchestra.run.smithi040.stdout: "statfs_resend": 0, 2023-07-14T11:29:23.686 INFO:teuthology.orchestra.run.smithi040.stdout: "command_active": 0, 2023-07-14T11:29:23.686 INFO:teuthology.orchestra.run.smithi040.stdout: "command_send": 0, 2023-07-14T11:29:23.686 INFO:teuthology.orchestra.run.smithi040.stdout: "command_resend": 0, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "map_epoch": 115, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "map_full": 0, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "map_inc": 25, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_sessions": 5, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_open": 5, 2023-07-14T11:29:23.687 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_session_close": 0, 2023-07-14T11:29:23.688 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_laggy": 0, 2023-07-14T11:29:23.688 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_wr": 0, 2023-07-14T11:29:23.688 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_rd": 2, 2023-07-14T11:29:23.689 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_del": 0 2023-07-14T11:29:23.689 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.689 INFO:teuthology.orchestra.run.smithi040.stdout: "oft": { 2023-07-14T11:29:23.689 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_objs": 0, 2023-07-14T11:29:23.689 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:23.690 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_updates": 0, 2023-07-14T11:29:23.690 INFO:teuthology.orchestra.run.smithi040.stdout: "omap_total_removes": 0 2023-07-14T11:29:23.690 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.690 INFO:teuthology.orchestra.run.smithi040.stdout: "osd-slow-ops": { 2023-07-14T11:29:23.690 INFO:teuthology.orchestra.run.smithi040.stdout: "slow_ops_count": 0 2023-07-14T11:29:23.691 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.691 INFO:teuthology.orchestra.run.smithi040.stdout: "purge_queue": { 2023-07-14T11:29:23.691 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:23.691 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:23.691 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing": 0, 2023-07-14T11:29:23.692 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:23.692 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_executed": 0, 2023-07-14T11:29:23.692 INFO:teuthology.orchestra.run.smithi040.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:23.692 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.693 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:23.693 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:23.693 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:23.693 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:23.694 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 4253, 2023-07-14T11:29:23.694 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 17694528, 2023-07-14T11:29:23.694 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:23.694 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 4253, 2023-07-14T11:29:23.694 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:23.695 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:23.695 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 4253, 2023-07-14T11:29:23.695 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 17694528, 2023-07-14T11:29:23.695 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:23.695 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.696 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.696 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.696 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.696 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.697 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:23.697 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:23.697 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:23.697 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:23.697 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 3661, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 14334168, 2023-07-14T11:29:23.698 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 3661, 2023-07-14T11:29:23.699 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 14334168, 2023-07-14T11:29:23.699 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:23.699 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.699 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.699 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.700 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.700 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.700 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:23.700 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:23.701 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 1024, 2023-07-14T11:29:23.701 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:23.701 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:23.701 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:23.701 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 3661, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 3661, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 3661, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 3661, 2023-07-14T11:29:23.702 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:23.703 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.703 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.703 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.703 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.703 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.704 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:23.704 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:23.704 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:23.704 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:23.704 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:23.705 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:23.705 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:23.705 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:23.705 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:23.705 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:23.706 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 0, 2023-07-14T11:29:23.706 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 0, 2023-07-14T11:29:23.706 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:23.706 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.707 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.707 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.707 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.707 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:23.708 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-write_buf_throttle-0x55876fef4500": { 2023-07-14T11:29:23.708 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:23.708 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 3758096384, 2023-07-14T11:29:23.708 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:23.708 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 0, 2023-07-14T11:29:23.709 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 0, 2023-07-14T11:29:23.709 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:23.709 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:23.709 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:23.709 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:23.710 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 0, 2023-07-14T11:29:23.710 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 0, 2023-07-14T11:29:23.710 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:23.710 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:23.710 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:23.711 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:23.711 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.711 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:23.712 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:23.988 INFO:tasks.check_counter:Daemon mds.g mds.exported=0 2023-07-14T11:29:23.989 INFO:tasks.check_counter:Daemon mds.g mds.imported=0 2023-07-14T11:29:23.989 INFO:tasks.check_counter:Daemon mds.g mds.root_rsnaps=6 2023-07-14T11:29:23.989 INFO:tasks.check_counter:Daemon mds.g mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:23.989 INFO:tasks.check_counter:Daemon mds.g mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:23.990 DEBUG:tasks.check_counter:Getting stats from j 2023-07-14T11:29:23.990 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.j perf dump 2023-07-14T11:29:24.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:23 smithi130 ceph-mon[129405]: pgmap v542: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:24.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:23 smithi040 ceph-mon[97954]: pgmap v542: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:24.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:23 smithi045 ceph-mon[128791]: pgmap v542: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.1 KiB/s rd, 102 B/s wr, 10 op/s 2023-07-14T11:29:24.722 INFO:teuthology.orchestra.run.smithi040.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.a/config 2023-07-14T11:29:25.949 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:25 smithi040 ceph-mon[97954]: pgmap v543: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:29:26.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:25 smithi130 ceph-mon[129405]: pgmap v543: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:29:26.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:25 smithi045 ceph-mon[128791]: pgmap v543: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.9 KiB/s rd, 9 op/s 2023-07-14T11:29:26.176 INFO:teuthology.orchestra.run.smithi040.stdout:{ 2023-07-14T11:29:26.177 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:26.177 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 0, 2023-07-14T11:29:26.177 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 2, 2023-07-14T11:29:26.177 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 0, 2023-07-14T11:29:26.177 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 0, 2023-07-14T11:29:26.178 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 4, 2023-07-14T11:29:26.178 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 0, 2023-07-14T11:29:26.178 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.003368438, 2023-07-14T11:29:26.178 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.000000000, 2023-07-14T11:29:26.178 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.000705714, 2023-07-14T11:29:26.179 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2023-07-14T11:29:26.179 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:26.179 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:26.179 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:26.179 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:26.180 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.180 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:26.180 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:26.180 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:26.180 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:26.181 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 331, 2023-07-14T11:29:26.182 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 322, 2023-07-14T11:29:26.182 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 1317168, 2023-07-14T11:29:26.182 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 184400, 2023-07-14T11:29:26.182 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 2, 2023-07-14T11:29:26.182 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 1, 2023-07-14T11:29:26.183 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.066077807, 2023-07-14T11:29:26.183 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.028087758, 2023-07-14T11:29:26.183 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.036534558, 2023-07-14T11:29:26.183 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.007001801, 2023-07-14T11:29:26.183 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:26.184 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 321, 2023-07-14T11:29:26.184 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.019040688, 2023-07-14T11:29:26.184 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000059316 2023-07-14T11:29:26.184 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.184 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 1317168, 2023-07-14T11:29:26.185 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 184400 2023-07-14T11:29:26.186 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.186 INFO:teuthology.orchestra.run.smithi040.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:26.186 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_messages": 6, 2023-07-14T11:29:26.186 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages": 248, 2023-07-14T11:29:26.186 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_bytes": 3264, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_bytes": 195056, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_total_time": 0.025197928, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_send_time": 0.017542880, 2023-07-14T11:29:26.187 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_recv_time": 0.003441422, 2023-07-14T11:29:26.188 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2023-07-14T11:29:26.188 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:26.188 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 246, 2023-07-14T11:29:26.188 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.019112418, 2023-07-14T11:29:26.188 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000077692 2023-07-14T11:29:26.189 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.189 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:26.189 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:26.189 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:26.189 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:26.190 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.190 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_recv_encrypted_bytes": 3264, 2023-07-14T11:29:26.190 INFO:teuthology.orchestra.run.smithi040.stdout: "msgr_send_encrypted_bytes": 195056 2023-07-14T11:29:26.190 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.190 INFO:teuthology.orchestra.run.smithi040.stdout: "cct": { 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: "total_workers": 0, 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: "unhealthy_workers": 0 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: "mempool": { 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:26.191 INFO:teuthology.orchestra.run.smithi040.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:26.192 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:26.192 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:26.192 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:26.192 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:26.192 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:26.193 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:26.193 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:26.193 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:26.193 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:26.193 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:26.194 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:26.194 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:26.194 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:26.194 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:26.194 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:26.195 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:26.195 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:26.195 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:26.195 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:26.195 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:26.196 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:26.197 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:26.197 INFO:teuthology.orchestra.run.smithi040.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:26.197 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:26.197 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_items": 0, 2023-07-14T11:29:26.197 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:26.198 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:26.198 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:26.198 INFO:teuthology.orchestra.run.smithi040.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:26.198 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_bytes": 148977, 2023-07-14T11:29:26.198 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_anon_items": 30, 2023-07-14T11:29:26.199 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:26.199 INFO:teuthology.orchestra.run.smithi040.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:26.199 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_bytes": 0, 2023-07-14T11:29:26.199 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_items": 0, 2023-07-14T11:29:26.199 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_bytes": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_items": 0, 2023-07-14T11:29:26.200 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:26.201 INFO:teuthology.orchestra.run.smithi040.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:26.201 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:26.201 INFO:teuthology.orchestra.run.smithi040.stdout: "pgmap_items": 0, 2023-07-14T11:29:26.201 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_bytes": 928, 2023-07-14T11:29:26.201 INFO:teuthology.orchestra.run.smithi040.stdout: "mds_co_items": 2, 2023-07-14T11:29:26.202 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:26.202 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_1_items": 0, 2023-07-14T11:29:26.202 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:26.202 INFO:teuthology.orchestra.run.smithi040.stdout: "unittest_2_items": 0 2023-07-14T11:29:26.202 INFO:teuthology.orchestra.run.smithi040.stdout: }, 2023-07-14T11:29:26.203 INFO:teuthology.orchestra.run.smithi040.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:26.203 INFO:teuthology.orchestra.run.smithi040.stdout: "val": 0, 2023-07-14T11:29:26.203 INFO:teuthology.orchestra.run.smithi040.stdout: "max": 104857600, 2023-07-14T11:29:26.203 INFO:teuthology.orchestra.run.smithi040.stdout: "get_started": 0, 2023-07-14T11:29:26.203 INFO:teuthology.orchestra.run.smithi040.stdout: "get": 333, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "get_sum": 1270390, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "get_or_fail_success": 333, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "take": 0, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "take_sum": 0, 2023-07-14T11:29:26.204 INFO:teuthology.orchestra.run.smithi040.stdout: "put": 333, 2023-07-14T11:29:26.205 INFO:teuthology.orchestra.run.smithi040.stdout: "put_sum": 1270390, 2023-07-14T11:29:26.205 INFO:teuthology.orchestra.run.smithi040.stdout: "wait": { 2023-07-14T11:29:26.205 INFO:teuthology.orchestra.run.smithi040.stdout: "avgcount": 0, 2023-07-14T11:29:26.205 INFO:teuthology.orchestra.run.smithi040.stdout: "sum": 0.000000000, 2023-07-14T11:29:26.205 INFO:teuthology.orchestra.run.smithi040.stdout: "avgtime": 0.000000000 2023-07-14T11:29:26.206 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:26.206 INFO:teuthology.orchestra.run.smithi040.stdout: } 2023-07-14T11:29:26.206 INFO:teuthology.orchestra.run.smithi040.stdout:} 2023-07-14T11:29:26.615 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.j 2023-07-14T11:29:26.615 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.j 2023-07-14T11:29:26.615 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.j 2023-07-14T11:29:26.616 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.j 2023-07-14T11:29:26.616 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.j 2023-07-14T11:29:26.616 DEBUG:tasks.check_counter:Getting stats from b 2023-07-14T11:29:26.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.b perf dump 2023-07-14T11:29:27.298 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:29:28.006 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:27 smithi045 ceph-mon[128791]: pgmap v544: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:27 smithi130 ceph-mon[129405]: pgmap v544: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:28.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:27 smithi040 ceph-mon[97954]: pgmap v544: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:28.895 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2023-07-14T11:29:28.895 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:28.895 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 5249341, 2023-07-14T11:29:28.896 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 5064146, 2023-07-14T11:29:28.896 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 834200925, 2023-07-14T11:29:28.896 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 875881919, 2023-07-14T11:29:28.896 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 25, 2023-07-14T11:29:28.897 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 4, 2023-07-14T11:29:28.897 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 183.527388393, 2023-07-14T11:29:28.897 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 71.075615194, 2023-07-14T11:29:28.897 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 127.909730373, 2023-07-14T11:29:28.897 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.289203713, 2023-07-14T11:29:28.898 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:28.898 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 5064149, 2023-07-14T11:29:28.898 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 58.778955276, 2023-07-14T11:29:28.898 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000011606 2023-07-14T11:29:28.898 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.899 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:28.899 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 10305328, 2023-07-14T11:29:28.899 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 1.548325109, 2023-07-14T11:29:28.899 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000150 2023-07-14T11:29:28.900 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.901 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 525792, 2023-07-14T11:29:28.901 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 167392 2023-07-14T11:29:28.902 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.902 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:28.903 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 285250, 2023-07-14T11:29:28.903 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 284996, 2023-07-14T11:29:28.904 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 49108856, 2023-07-14T11:29:28.904 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 51875390, 2023-07-14T11:29:28.905 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 20, 2023-07-14T11:29:28.905 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 6, 2023-07-14T11:29:28.906 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 9.244079774, 2023-07-14T11:29:28.906 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 3.506370158, 2023-07-14T11:29:28.907 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 6.305284983, 2023-07-14T11:29:28.907 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.060755658, 2023-07-14T11:29:28.907 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:28.908 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 284997, 2023-07-14T11:29:28.908 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 3.116257462, 2023-07-14T11:29:28.909 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000010934 2023-07-14T11:29:28.909 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.909 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:28.910 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 566108, 2023-07-14T11:29:28.910 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.050417328, 2023-07-14T11:29:28.910 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000089 2023-07-14T11:29:28.910 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.911 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 2785696, 2023-07-14T11:29:28.911 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 833648 2023-07-14T11:29:28.911 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.911 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:28.911 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 4408504, 2023-07-14T11:29:28.912 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 4410669, 2023-07-14T11:29:28.912 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 699471254, 2023-07-14T11:29:28.913 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 735673021, 2023-07-14T11:29:28.913 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 90, 2023-07-14T11:29:28.913 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 13, 2023-07-14T11:29:28.913 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 195.097225145, 2023-07-14T11:29:28.913 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 84.232651828, 2023-07-14T11:29:28.914 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 138.814857319, 2023-07-14T11:29:28.914 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.391123749, 2023-07-14T11:29:28.914 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:28.914 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 4410660, 2023-07-14T11:29:28.914 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 45.626857512, 2023-07-14T11:29:28.915 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000010344 2023-07-14T11:29:28.915 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.915 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:28.915 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 8814186, 2023-07-14T11:29:28.915 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 1.097294810, 2023-07-14T11:29:28.916 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000124 2023-07-14T11:29:28.916 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.916 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 43056, 2023-07-14T11:29:28.916 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 136448 2023-07-14T11:29:28.917 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.917 INFO:teuthology.orchestra.run.smithi045.stdout: "cct": { 2023-07-14T11:29:28.917 INFO:teuthology.orchestra.run.smithi045.stdout: "total_workers": 1, 2023-07-14T11:29:28.917 INFO:teuthology.orchestra.run.smithi045.stdout: "unhealthy_workers": 0 2023-07-14T11:29:28.917 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.918 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-MDSRank": { 2023-07-14T11:29:28.918 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:28.918 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:28.918 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 3423, 2023-07-14T11:29:28.919 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 3.505147857, 2023-07-14T11:29:28.919 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.001023998 2023-07-14T11:29:28.919 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:28.919 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.920 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:28.920 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:28.920 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:28.920 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 194, 2023-07-14T11:29:28.920 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.010209371, 2023-07-14T11:29:28.921 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000052625 2023-07-14T11:29:28.921 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:28.921 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.921 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2023-07-14T11:29:28.921 INFO:teuthology.orchestra.run.smithi045.stdout: "request": 2667, 2023-07-14T11:29:28.922 INFO:teuthology.orchestra.run.smithi045.stdout: "reply": 2316, 2023-07-14T11:29:28.922 INFO:teuthology.orchestra.run.smithi045.stdout: "reply_latency": { 2023-07-14T11:29:28.922 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 2316, 2023-07-14T11:29:28.922 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 61.801736000, 2023-07-14T11:29:28.923 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.026684687 2023-07-14T11:29:28.923 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.923 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_reply": 0, 2023-07-14T11:29:28.923 INFO:teuthology.orchestra.run.smithi045.stdout: "forward": 351, 2023-07-14T11:29:28.924 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:28.924 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:28.924 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_commit": 39, 2023-07-14T11:29:28.924 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_split": 3, 2023-07-14T11:29:28.925 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_merge": 0, 2023-07-14T11:29:28.925 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes": 678, 2023-07-14T11:29:28.925 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_top": 49, 2023-07-14T11:29:28.926 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_bottom": 447, 2023-07-14T11:29:28.926 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pin_tail": 182, 2023-07-14T11:29:28.926 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pinned": 609, 2023-07-14T11:29:28.926 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_expired": 1211, 2023-07-14T11:29:28.926 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_with_caps": 129, 2023-07-14T11:29:28.927 INFO:teuthology.orchestra.run.smithi045.stdout: "caps": 130, 2023-07-14T11:29:28.927 INFO:teuthology.orchestra.run.smithi045.stdout: "subtrees": 129, 2023-07-14T11:29:28.927 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse": 4329, 2023-07-14T11:29:28.928 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_hit": 3153, 2023-07-14T11:29:28.928 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_forward": 202, 2023-07-14T11:29:28.928 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_discover": 25, 2023-07-14T11:29:28.929 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:28.929 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:28.929 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_lock": 326, 2023-07-14T11:29:28.930 INFO:teuthology.orchestra.run.smithi045.stdout: "load_cent": 1000, 2023-07-14T11:29:28.930 INFO:teuthology.orchestra.run.smithi045.stdout: "q": 1, 2023-07-14T11:29:28.930 INFO:teuthology.orchestra.run.smithi045.stdout: "exported": 21, 2023-07-14T11:29:28.930 INFO:teuthology.orchestra.run.smithi045.stdout: "exported_inodes": 1151, 2023-07-14T11:29:28.931 INFO:teuthology.orchestra.run.smithi045.stdout: "imported": 7, 2023-07-14T11:29:28.931 INFO:teuthology.orchestra.run.smithi045.stdout: "imported_inodes": 126, 2023-07-14T11:29:28.931 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:28.931 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_backtrace_fetch": 3015, 2023-07-14T11:29:28.931 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_peer_discover": 2, 2023-07-14T11:29:28.932 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rfiles": 1028, 2023-07-14T11:29:28.932 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rbytes": 5852694910, 2023-07-14T11:29:28.932 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rsnaps": 6, 2023-07-14T11:29:28.932 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_fetch": 3015, 2023-07-14T11:29:28.932 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:28.933 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:28.933 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:28.933 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_inodes": 113, 2023-07-14T11:29:28.933 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_base_inodes": 5, 2023-07-14T11:29:28.933 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dirfrag_rstats": 108, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_file_inodes": 2902, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_inode_file_caps": 22, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_revoke": 542, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_grant": 2112, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:28.934 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:28.935 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:28.935 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps": 2100, 2023-07-14T11:29:28.935 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps_dirty": 1139, 2023-07-14T11:29:28.935 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_cap_release": 48, 2023-07-14T11:29:28.935 INFO:teuthology.orchestra.run.smithi045.stdout: "process_request_cap_release": 439 2023-07-14T11:29:28.936 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.936 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_cache": { 2023-07-14T11:29:28.936 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update": 0, 2023-07-14T11:29:28.936 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:28.936 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_try_discover": 0, 2023-07-14T11:29:28.937 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_send_discover": 26, 2023-07-14T11:29:28.937 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_handle_discover": 33, 2023-07-14T11:29:28.937 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays": 0, 2023-07-14T11:29:28.937 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:28.937 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:28.938 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_created": 81, 2023-07-14T11:29:28.938 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_enqueued": 47, 2023-07-14T11:29:28.938 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_reintegrated": 143, 2023-07-14T11:29:28.938 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_migrated": 0, 2023-07-14T11:29:28.938 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:28.939 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:28.939 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:28.939 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_started": 0, 2023-07-14T11:29:28.939 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_completed": 0, 2023-07-14T11:29:28.939 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_enqueue_scrub": 5, 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_exportdir": 21, 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_flush": 0, 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragmentdir": 3, 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_inodestats": 0 2023-07-14T11:29:28.940 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_log": { 2023-07-14T11:29:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: "evadd": 4334, 2023-07-14T11:29:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: "evex": 0, 2023-07-14T11:29:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: "evtrm": 0, 2023-07-14T11:29:28.941 INFO:teuthology.orchestra.run.smithi045.stdout: "ev": 4334, 2023-07-14T11:29:28.942 INFO:teuthology.orchestra.run.smithi045.stdout: "evexg": 0, 2023-07-14T11:29:28.942 INFO:teuthology.orchestra.run.smithi045.stdout: "evexd": 0, 2023-07-14T11:29:28.942 INFO:teuthology.orchestra.run.smithi045.stdout: "segadd": 7, 2023-07-14T11:29:28.942 INFO:teuthology.orchestra.run.smithi045.stdout: "segex": 0, 2023-07-14T11:29:28.942 INFO:teuthology.orchestra.run.smithi045.stdout: "segtrm": 0, 2023-07-14T11:29:28.943 INFO:teuthology.orchestra.run.smithi045.stdout: "seg": 7, 2023-07-14T11:29:28.943 INFO:teuthology.orchestra.run.smithi045.stdout: "segexg": 0, 2023-07-14T11:29:28.943 INFO:teuthology.orchestra.run.smithi045.stdout: "segexd": 0, 2023-07-14T11:29:28.943 INFO:teuthology.orchestra.run.smithi045.stdout: "expos": 4194304, 2023-07-14T11:29:28.943 INFO:teuthology.orchestra.run.smithi045.stdout: "wrpos": 17341617, 2023-07-14T11:29:28.944 INFO:teuthology.orchestra.run.smithi045.stdout: "rdpos": 0, 2023-07-14T11:29:28.944 INFO:teuthology.orchestra.run.smithi045.stdout: "jlat": { 2023-07-14T11:29:28.944 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1630, 2023-07-14T11:29:28.944 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 25.326768928, 2023-07-14T11:29:28.944 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.015537895 2023-07-14T11:29:28.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.945 INFO:teuthology.orchestra.run.smithi045.stdout: "replayed": 0 2023-07-14T11:29:28.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.945 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_mem": { 2023-07-14T11:29:28.945 INFO:teuthology.orchestra.run.smithi045.stdout: "ino": 587, 2023-07-14T11:29:28.946 INFO:teuthology.orchestra.run.smithi045.stdout: "ino+": 1782, 2023-07-14T11:29:28.946 INFO:teuthology.orchestra.run.smithi045.stdout: "ino-": 1195, 2023-07-14T11:29:28.946 INFO:teuthology.orchestra.run.smithi045.stdout: "dir": 174, 2023-07-14T11:29:28.946 INFO:teuthology.orchestra.run.smithi045.stdout: "dir+": 201, 2023-07-14T11:29:28.947 INFO:teuthology.orchestra.run.smithi045.stdout: "dir-": 27, 2023-07-14T11:29:28.947 INFO:teuthology.orchestra.run.smithi045.stdout: "dn": 678, 2023-07-14T11:29:28.947 INFO:teuthology.orchestra.run.smithi045.stdout: "dn+": 2245, 2023-07-14T11:29:28.947 INFO:teuthology.orchestra.run.smithi045.stdout: "dn-": 1567, 2023-07-14T11:29:28.948 INFO:teuthology.orchestra.run.smithi045.stdout: "cap": 130, 2023-07-14T11:29:28.948 INFO:teuthology.orchestra.run.smithi045.stdout: "cap+": 1745, 2023-07-14T11:29:28.948 INFO:teuthology.orchestra.run.smithi045.stdout: "cap-": 1615, 2023-07-14T11:29:28.948 INFO:teuthology.orchestra.run.smithi045.stdout: "rss": 72084, 2023-07-14T11:29:28.948 INFO:teuthology.orchestra.run.smithi045.stdout: "heap": 207156 2023-07-14T11:29:28.949 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.949 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_server": { 2023-07-14T11:29:28.949 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_client_request": 3532, 2023-07-14T11:29:28.949 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_server_request": 2476, 2023-07-14T11:29:28.950 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_request": 2667, 2023-07-14T11:29:28.950 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_session": 335, 2023-07-14T11:29:28.950 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_peer_request": 4957, 2023-07-14T11:29:28.950 INFO:teuthology.orchestra.run.smithi045.stdout: "req_create_latency": { 2023-07-14T11:29:28.951 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1040, 2023-07-14T11:29:28.951 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 5.720722504, 2023-07-14T11:29:28.951 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.005500694 2023-07-14T11:29:28.951 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.952 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getattr_latency": { 2023-07-14T11:29:28.952 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 221, 2023-07-14T11:29:28.952 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 6.457673809, 2023-07-14T11:29:28.952 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.029220243 2023-07-14T11:29:28.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.953 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:28.953 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.953 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.953 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.954 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.954 INFO:teuthology.orchestra.run.smithi045.stdout: "req_link_latency": { 2023-07-14T11:29:28.954 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 34, 2023-07-14T11:29:28.954 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.085924348, 2023-07-14T11:29:28.955 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.002527186 2023-07-14T11:29:28.955 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.955 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookup_latency": { 2023-07-14T11:29:28.956 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 98, 2023-07-14T11:29:28.956 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.196873140, 2023-07-14T11:29:28.956 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.002008909 2023-07-14T11:29:28.956 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.957 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:28.957 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.957 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.957 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.957 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.958 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupino_latency": { 2023-07-14T11:29:28.958 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.958 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.958 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.958 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.959 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupname_latency": { 2023-07-14T11:29:28.959 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.959 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.959 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.959 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.960 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:28.960 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.960 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.960 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.960 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.961 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:28.961 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.961 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.961 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.962 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.962 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lssnap_latency": { 2023-07-14T11:29:28.962 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 17, 2023-07-14T11:29:28.962 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.004230119, 2023-07-14T11:29:28.962 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000248830 2023-07-14T11:29:28.963 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.963 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mkdir_latency": { 2023-07-14T11:29:28.963 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 28, 2023-07-14T11:29:28.963 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.784596462, 2023-07-14T11:29:28.963 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.028021302 2023-07-14T11:29:28.964 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.964 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mknod_latency": { 2023-07-14T11:29:28.964 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.964 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.965 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.965 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.965 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mksnap_latency": { 2023-07-14T11:29:28.965 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 17, 2023-07-14T11:29:28.965 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 6.243781526, 2023-07-14T11:29:28.966 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.367281266 2023-07-14T11:29:28.966 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.966 INFO:teuthology.orchestra.run.smithi045.stdout: "req_open_latency": { 2023-07-14T11:29:28.966 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 523, 2023-07-14T11:29:28.966 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.220244850, 2023-07-14T11:29:28.967 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000421118 2023-07-14T11:29:28.967 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.967 INFO:teuthology.orchestra.run.smithi045.stdout: "req_readdir_latency": { 2023-07-14T11:29:28.967 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 88, 2023-07-14T11:29:28.967 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 2.160003653, 2023-07-14T11:29:28.968 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.024545496 2023-07-14T11:29:28.968 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.968 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rename_latency": { 2023-07-14T11:29:28.968 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 177, 2023-07-14T11:29:28.968 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 37.061148381, 2023-07-14T11:29:28.969 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.209385019 2023-07-14T11:29:28.969 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.969 INFO:teuthology.orchestra.run.smithi045.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:28.969 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.969 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.970 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.970 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.970 INFO:teuthology.orchestra.run.smithi045.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:28.970 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.970 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.971 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.971 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmdir_latency": { 2023-07-14T11:29:28.971 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.971 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.972 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.972 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.972 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:28.972 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 11, 2023-07-14T11:29:28.973 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 2.624609117, 2023-07-14T11:29:28.973 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.238600828 2023-07-14T11:29:28.973 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.973 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:28.974 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.974 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.974 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.974 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.975 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setattr_latency": { 2023-07-14T11:29:28.975 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1, 2023-07-14T11:29:28.975 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000721420, 2023-07-14T11:29:28.975 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000721420 2023-07-14T11:29:28.975 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.976 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:28.976 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.976 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.976 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.976 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.977 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:28.977 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.977 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.977 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.977 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.978 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setlayout_latency": { 2023-07-14T11:29:28.978 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.978 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.978 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.979 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.979 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setxattr_latency": { 2023-07-14T11:29:28.979 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1, 2023-07-14T11:29:28.979 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.002905466, 2023-07-14T11:29:28.979 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.002905466 2023-07-14T11:29:28.980 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.980 INFO:teuthology.orchestra.run.smithi045.stdout: "req_symlink_latency": { 2023-07-14T11:29:28.980 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.980 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.980 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.981 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.981 INFO:teuthology.orchestra.run.smithi045.stdout: "req_unlink_latency": { 2023-07-14T11:29:28.981 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 60, 2023-07-14T11:29:28.981 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.238301205, 2023-07-14T11:29:28.982 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.003971686 2023-07-14T11:29:28.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.982 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:28.982 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:28.982 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:28.983 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:28.983 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:28.983 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:28.983 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:28.983 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.984 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_sessions": { 2023-07-14T11:29:28.984 INFO:teuthology.orchestra.run.smithi045.stdout: "session_count": 2, 2023-07-14T11:29:28.984 INFO:teuthology.orchestra.run.smithi045.stdout: "session_add": 4, 2023-07-14T11:29:28.984 INFO:teuthology.orchestra.run.smithi045.stdout: "session_remove": 2, 2023-07-14T11:29:28.985 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_open": 2, 2023-07-14T11:29:28.985 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_stale": 0, 2023-07-14T11:29:28.985 INFO:teuthology.orchestra.run.smithi045.stdout: "total_load": 28, 2023-07-14T11:29:28.985 INFO:teuthology.orchestra.run.smithi045.stdout: "average_load": 14, 2023-07-14T11:29:28.986 INFO:teuthology.orchestra.run.smithi045.stdout: "avg_session_uptime": 1023 2023-07-14T11:29:28.986 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.986 INFO:teuthology.orchestra.run.smithi045.stdout: "mempool": { 2023-07-14T11:29:28.986 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:28.986 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:28.987 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:28.987 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:28.987 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:28.987 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:28.988 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:28.988 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:28.988 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:28.988 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:28.989 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:28.989 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:28.989 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:28.989 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:28.990 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:28.990 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:28.990 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:28.990 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:28.991 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:28.991 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:28.991 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:28.991 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:28.991 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:28.992 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:28.992 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:28.992 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:28.992 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:28.992 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:28.993 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:28.993 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:28.993 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:28.993 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_items": 0, 2023-07-14T11:29:28.993 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:28.994 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:28.994 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:28.994 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:28.994 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_bytes": 475914, 2023-07-14T11:29:28.994 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_items": 521, 2023-07-14T11:29:28.995 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_bytes": 440, 2023-07-14T11:29:28.995 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_items": 5, 2023-07-14T11:29:28.995 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_bytes": 0, 2023-07-14T11:29:28.995 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_items": 0, 2023-07-14T11:29:28.995 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:28.996 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:28.996 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:28.996 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:28.996 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:28.996 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_items": 183, 2023-07-14T11:29:28.997 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:28.997 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:28.997 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:28.997 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_items": 0, 2023-07-14T11:29:28.997 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_bytes": 2201936, 2023-07-14T11:29:28.998 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_items": 23801, 2023-07-14T11:29:28.998 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:28.998 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_items": 0, 2023-07-14T11:29:28.998 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:28.998 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_items": 0 2023-07-14T11:29:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: "objecter": { 2023-07-14T11:29:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: "op_active": 0, 2023-07-14T11:29:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: "op_laggy": 0, 2023-07-14T11:29:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send": 4985, 2023-07-14T11:29:29.000 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send_bytes": 13626477, 2023-07-14T11:29:29.000 INFO:teuthology.orchestra.run.smithi045.stdout: "op_resend": 31, 2023-07-14T11:29:29.000 INFO:teuthology.orchestra.run.smithi045.stdout: "op_reply": 4985, 2023-07-14T11:29:29.000 INFO:teuthology.orchestra.run.smithi045.stdout: "op_latency": { 2023-07-14T11:29:29.000 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 4985, 2023-07-14T11:29:29.001 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 10.197830244, 2023-07-14T11:29:29.001 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.002045703 2023-07-14T11:29:29.001 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.001 INFO:teuthology.orchestra.run.smithi045.stdout: "op_inflight": 0, 2023-07-14T11:29:29.002 INFO:teuthology.orchestra.run.smithi045.stdout: "oplen_avg": { 2023-07-14T11:29:29.002 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 4985, 2023-07-14T11:29:29.002 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 5103 2023-07-14T11:29:29.002 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.002 INFO:teuthology.orchestra.run.smithi045.stdout: "op": 4985, 2023-07-14T11:29:29.003 INFO:teuthology.orchestra.run.smithi045.stdout: "op_r": 3050, 2023-07-14T11:29:29.003 INFO:teuthology.orchestra.run.smithi045.stdout: "op_w": 1935, 2023-07-14T11:29:29.003 INFO:teuthology.orchestra.run.smithi045.stdout: "op_rmw": 0, 2023-07-14T11:29:29.003 INFO:teuthology.orchestra.run.smithi045.stdout: "op_pg": 0, 2023-07-14T11:29:29.003 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_stat": 0, 2023-07-14T11:29:29.004 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_create": 11, 2023-07-14T11:29:29.004 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_read": 30, 2023-07-14T11:29:29.004 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_write": 1655, 2023-07-14T11:29:29.004 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writefull": 64, 2023-07-14T11:29:29.004 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writesame": 0, 2023-07-14T11:29:29.005 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_append": 0, 2023-07-14T11:29:29.005 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_zero": 0, 2023-07-14T11:29:29.005 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_truncate": 3, 2023-07-14T11:29:29.005 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_delete": 109, 2023-07-14T11:29:29.005 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_mapext": 0, 2023-07-14T11:29:29.006 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:29.006 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:29.006 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_getxattr": 3020, 2023-07-14T11:29:29.006 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_setxattr": 22, 2023-07-14T11:29:29.006 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:29.007 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:29.007 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:29.007 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_call": 0, 2023-07-14T11:29:29.007 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_watch": 0, 2023-07-14T11:29:29.008 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_notify": 0, 2023-07-14T11:29:29.008 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:29.008 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls": 0, 2023-07-14T11:29:29.008 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:29.009 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_other": 0, 2023-07-14T11:29:29.009 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_active": 0, 2023-07-14T11:29:29.009 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_send": 0, 2023-07-14T11:29:29.009 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_resend": 0, 2023-07-14T11:29:29.009 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_ping": 0, 2023-07-14T11:29:29.010 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_active": 0, 2023-07-14T11:29:29.010 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_send": 0, 2023-07-14T11:29:29.010 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_resend": 0, 2023-07-14T11:29:29.010 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_active": 0, 2023-07-14T11:29:29.011 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_send": 0, 2023-07-14T11:29:29.011 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_resend": 0, 2023-07-14T11:29:29.011 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_active": 0, 2023-07-14T11:29:29.011 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_send": 0, 2023-07-14T11:29:29.011 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_resend": 0, 2023-07-14T11:29:29.012 INFO:teuthology.orchestra.run.smithi045.stdout: "command_active": 0, 2023-07-14T11:29:29.012 INFO:teuthology.orchestra.run.smithi045.stdout: "command_send": 0, 2023-07-14T11:29:29.012 INFO:teuthology.orchestra.run.smithi045.stdout: "command_resend": 0, 2023-07-14T11:29:29.012 INFO:teuthology.orchestra.run.smithi045.stdout: "map_epoch": 115, 2023-07-14T11:29:29.012 INFO:teuthology.orchestra.run.smithi045.stdout: "map_full": 0, 2023-07-14T11:29:29.013 INFO:teuthology.orchestra.run.smithi045.stdout: "map_inc": 32, 2023-07-14T11:29:29.013 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_sessions": 12, 2023-07-14T11:29:29.013 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_open": 12, 2023-07-14T11:29:29.013 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_close": 0, 2023-07-14T11:29:29.013 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_laggy": 0, 2023-07-14T11:29:29.014 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_wr": 154, 2023-07-14T11:29:29.014 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_rd": 0, 2023-07-14T11:29:29.014 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_del": 35 2023-07-14T11:29:29.014 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.015 INFO:teuthology.orchestra.run.smithi045.stdout: "oft": { 2023-07-14T11:29:29.015 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_objs": 1, 2023-07-14T11:29:29.015 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:29.015 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_updates": 0, 2023-07-14T11:29:29.016 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_removes": 0 2023-07-14T11:29:29.016 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.016 INFO:teuthology.orchestra.run.smithi045.stdout: "osd-slow-ops": { 2023-07-14T11:29:29.017 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_ops_count": 5158 2023-07-14T11:29:29.017 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.017 INFO:teuthology.orchestra.run.smithi045.stdout: "purge_queue": { 2023-07-14T11:29:29.017 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:29.018 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops_high_water": 10, 2023-07-14T11:29:29.018 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing": 0, 2023-07-14T11:29:29.018 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_high_water": 5, 2023-07-14T11:29:29.018 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executed": 47, 2023-07-14T11:29:29.019 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:29.019 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.019 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:29.019 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 68, 2023-07-14T11:29:29.020 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:29.020 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:29.020 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 9943095, 2023-07-14T11:29:29.020 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 687804962, 2023-07-14T11:29:29.020 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 9943095, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 9943094, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 687804894, 2023-07-14T11:29:29.021 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:29.022 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:29.022 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:29.022 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:29.022 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.022 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.023 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:29.023 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:29.023 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:29.023 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:29.023 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:29.024 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:29.024 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:29.024 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:29.024 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 4985, 2023-07-14T11:29:29.024 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 13631768, 2023-07-14T11:29:29.025 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 4985, 2023-07-14T11:29:29.025 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 13631768, 2023-07-14T11:29:29.025 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:29.025 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:29.025 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:29.026 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:29.026 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.026 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.026 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:29.026 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:29.027 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 1024, 2023-07-14T11:29:29.027 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:29.027 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:29.027 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:29.027 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:29.028 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:29.028 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 4985, 2023-07-14T11:29:29.028 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 4985, 2023-07-14T11:29:29.028 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 4985, 2023-07-14T11:29:29.028 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 4985, 2023-07-14T11:29:29.029 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:29.029 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:29.029 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:29.029 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:29.029 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:29.030 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 47, 2023-07-14T11:29:29.031 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 5291, 2023-07-14T11:29:29.031 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:29.031 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 47, 2023-07-14T11:29:29.031 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:29.031 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:29.032 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 25, 2023-07-14T11:29:29.032 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 5291, 2023-07-14T11:29:29.032 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:29.032 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:29.032 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:29.033 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:29.033 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.033 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:29.033 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle-0x555b7e6800a0": { 2023-07-14T11:29:29.034 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:29.034 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:29.034 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:29.034 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 4334, 2023-07-14T11:29:29.034 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 13147313, 2023-07-14T11:29:29.035 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:29.035 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 4334, 2023-07-14T11:29:29.035 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:29.035 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:29.036 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 1630, 2023-07-14T11:29:29.036 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 13147313, 2023-07-14T11:29:29.036 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:29.037 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:29.037 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:29.037 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:29.037 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.038 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:29.038 INFO:teuthology.orchestra.run.smithi045.stdout:} 2023-07-14T11:29:29.274 INFO:tasks.check_counter:Daemon mds.b mds.exported=21 2023-07-14T11:29:29.283 INFO:tasks.check_counter:Daemon mds.b mds.imported=7 2023-07-14T11:29:29.283 INFO:tasks.check_counter:Daemon mds.b mds.root_rsnaps=6 2023-07-14T11:29:29.283 INFO:tasks.check_counter:Daemon mds.b mds_server.req_mksnap_latency.avgcount=17 2023-07-14T11:29:29.284 INFO:tasks.check_counter:Daemon mds.b mds_server.req_rmsnap_latency.avgcount=11 2023-07-14T11:29:29.284 DEBUG:tasks.check_counter:Getting stats from e 2023-07-14T11:29:29.284 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.e perf dump 2023-07-14T11:29:29.935 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:29:30.145 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:29 smithi045 ceph-mon[128791]: pgmap v545: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:30.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:29 smithi130 ceph-mon[129405]: pgmap v545: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:30.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:29 smithi040 ceph-mon[97954]: pgmap v545: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:31.274 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2023-07-14T11:29:31.275 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:31.275 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 587, 2023-07-14T11:29:31.275 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 353, 2023-07-14T11:29:31.276 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 2723170, 2023-07-14T11:29:31.276 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 189933, 2023-07-14T11:29:31.276 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 6, 2023-07-14T11:29:31.276 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 1, 2023-07-14T11:29:31.276 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.062150693, 2023-07-14T11:29:31.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.015013642, 2023-07-14T11:29:31.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.051417597, 2023-07-14T11:29:31.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.008325796, 2023-07-14T11:29:31.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:31.277 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 352, 2023-07-14T11:29:31.278 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.011880980, 2023-07-14T11:29:31.278 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000033752 2023-07-14T11:29:31.278 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.279 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:31.279 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.279 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.279 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.280 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 2722336, 2023-07-14T11:29:31.280 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 189616 2023-07-14T11:29:31.280 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.280 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:31.281 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 1044, 2023-07-14T11:29:31.281 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 1288, 2023-07-14T11:29:31.281 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 256674, 2023-07-14T11:29:31.282 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 891683, 2023-07-14T11:29:31.282 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 7, 2023-07-14T11:29:31.282 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:31.283 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.093310172, 2023-07-14T11:29:31.283 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.025884180, 2023-07-14T11:29:31.283 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.018917135, 2023-07-14T11:29:31.283 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.038854570, 2023-07-14T11:29:31.284 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:31.284 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1285, 2023-07-14T11:29:31.284 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.050284160, 2023-07-14T11:29:31.284 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000039131 2023-07-14T11:29:31.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.285 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:31.285 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.285 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.285 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.285 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 320, 2023-07-14T11:29:31.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 567616 2023-07-14T11:29:31.286 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.286 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:31.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 2422, 2023-07-14T11:29:31.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 2422, 2023-07-14T11:29:31.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 3675983, 2023-07-14T11:29:31.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 752504, 2023-07-14T11:29:31.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 4, 2023-07-14T11:29:31.288 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:31.288 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.191501634, 2023-07-14T11:29:31.288 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.057569800, 2023-07-14T11:29:31.288 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.047395367, 2023-07-14T11:29:31.289 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.069918384, 2023-07-14T11:29:31.289 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:31.289 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 2421, 2023-07-14T11:29:31.289 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.061815869, 2023-07-14T11:29:31.290 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000025533 2023-07-14T11:29:31.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.290 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:31.290 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.290 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.291 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.291 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.291 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 2944, 2023-07-14T11:29:31.292 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 448 2023-07-14T11:29:31.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.292 INFO:teuthology.orchestra.run.smithi045.stdout: "cct": { 2023-07-14T11:29:31.293 INFO:teuthology.orchestra.run.smithi045.stdout: "total_workers": 1, 2023-07-14T11:29:31.293 INFO:teuthology.orchestra.run.smithi045.stdout: "unhealthy_workers": 0 2023-07-14T11:29:31.293 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.294 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-MDSRank": { 2023-07-14T11:29:31.294 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:31.294 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:31.294 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 5772, 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.159622281, 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000027654 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:31.295 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:31.296 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:31.296 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.296 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.296 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.297 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.297 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.297 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2023-07-14T11:29:31.297 INFO:teuthology.orchestra.run.smithi045.stdout: "request": 0, 2023-07-14T11:29:31.297 INFO:teuthology.orchestra.run.smithi045.stdout: "reply": 0, 2023-07-14T11:29:31.298 INFO:teuthology.orchestra.run.smithi045.stdout: "reply_latency": { 2023-07-14T11:29:31.298 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.298 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.298 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.298 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.299 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_reply": 0, 2023-07-14T11:29:31.299 INFO:teuthology.orchestra.run.smithi045.stdout: "forward": 0, 2023-07-14T11:29:31.299 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:31.299 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:31.300 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_commit": 0, 2023-07-14T11:29:31.300 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_split": 0, 2023-07-14T11:29:31.300 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_merge": 0, 2023-07-14T11:29:31.300 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes": 853, 2023-07-14T11:29:31.300 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_top": 42, 2023-07-14T11:29:31.301 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_bottom": 160, 2023-07-14T11:29:31.301 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pin_tail": 651, 2023-07-14T11:29:31.301 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pinned": 793, 2023-07-14T11:29:31.301 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_expired": 26, 2023-07-14T11:29:31.301 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_with_caps": 0, 2023-07-14T11:29:31.302 INFO:teuthology.orchestra.run.smithi045.stdout: "caps": 0, 2023-07-14T11:29:31.302 INFO:teuthology.orchestra.run.smithi045.stdout: "subtrees": 46, 2023-07-14T11:29:31.302 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse": 0, 2023-07-14T11:29:31.303 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_hit": 0, 2023-07-14T11:29:31.303 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_forward": 0, 2023-07-14T11:29:31.303 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_discover": 0, 2023-07-14T11:29:31.304 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:31.304 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:31.304 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_lock": 0, 2023-07-14T11:29:31.305 INFO:teuthology.orchestra.run.smithi045.stdout: "load_cent": 0, 2023-07-14T11:29:31.305 INFO:teuthology.orchestra.run.smithi045.stdout: "q": 0, 2023-07-14T11:29:31.305 INFO:teuthology.orchestra.run.smithi045.stdout: "exported": 0, 2023-07-14T11:29:31.305 INFO:teuthology.orchestra.run.smithi045.stdout: "exported_inodes": 0, 2023-07-14T11:29:31.305 INFO:teuthology.orchestra.run.smithi045.stdout: "imported": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "imported_inodes": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_backtrace_fetch": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rfiles": 0, 2023-07-14T11:29:31.306 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rbytes": 0, 2023-07-14T11:29:31.307 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rsnaps": 0, 2023-07-14T11:29:31.307 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_fetch": 0, 2023-07-14T11:29:31.307 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:31.307 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:31.307 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:31.308 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:31.308 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:31.308 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:31.308 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_file_inodes": 0, 2023-07-14T11:29:31.309 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_inode_file_caps": 0, 2023-07-14T11:29:31.309 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_revoke": 0, 2023-07-14T11:29:31.309 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_grant": 0, 2023-07-14T11:29:31.309 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:31.309 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:31.310 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:31.310 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps": 0, 2023-07-14T11:29:31.310 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps_dirty": 0, 2023-07-14T11:29:31.310 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_cap_release": 0, 2023-07-14T11:29:31.311 INFO:teuthology.orchestra.run.smithi045.stdout: "process_request_cap_release": 0 2023-07-14T11:29:31.311 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.311 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_cache": { 2023-07-14T11:29:31.311 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update": 0, 2023-07-14T11:29:31.311 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:31.312 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_try_discover": 0, 2023-07-14T11:29:31.312 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_send_discover": 0, 2023-07-14T11:29:31.312 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_handle_discover": 0, 2023-07-14T11:29:31.312 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays": 0, 2023-07-14T11:29:31.312 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:31.313 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:31.313 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_created": 0, 2023-07-14T11:29:31.313 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_enqueued": 0, 2023-07-14T11:29:31.314 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_reintegrated": 0, 2023-07-14T11:29:31.314 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_migrated": 0, 2023-07-14T11:29:31.314 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:31.315 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:31.315 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:31.315 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_started": 0, 2023-07-14T11:29:31.315 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_completed": 0, 2023-07-14T11:29:31.316 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:31.316 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_exportdir": 0, 2023-07-14T11:29:31.316 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_flush": 0, 2023-07-14T11:29:31.316 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragmentdir": 0, 2023-07-14T11:29:31.316 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:31.317 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_inodestats": 0 2023-07-14T11:29:31.317 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.317 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_log": { 2023-07-14T11:29:31.317 INFO:teuthology.orchestra.run.smithi045.stdout: "evadd": 0, 2023-07-14T11:29:31.317 INFO:teuthology.orchestra.run.smithi045.stdout: "evex": 0, 2023-07-14T11:29:31.318 INFO:teuthology.orchestra.run.smithi045.stdout: "evtrm": 0, 2023-07-14T11:29:31.318 INFO:teuthology.orchestra.run.smithi045.stdout: "ev": 0, 2023-07-14T11:29:31.318 INFO:teuthology.orchestra.run.smithi045.stdout: "evexg": 0, 2023-07-14T11:29:31.318 INFO:teuthology.orchestra.run.smithi045.stdout: "evexd": 0, 2023-07-14T11:29:31.318 INFO:teuthology.orchestra.run.smithi045.stdout: "segadd": 0, 2023-07-14T11:29:31.319 INFO:teuthology.orchestra.run.smithi045.stdout: "segex": 0, 2023-07-14T11:29:31.319 INFO:teuthology.orchestra.run.smithi045.stdout: "segtrm": 0, 2023-07-14T11:29:31.319 INFO:teuthology.orchestra.run.smithi045.stdout: "seg": 2, 2023-07-14T11:29:31.319 INFO:teuthology.orchestra.run.smithi045.stdout: "segexg": 0, 2023-07-14T11:29:31.319 INFO:teuthology.orchestra.run.smithi045.stdout: "segexd": 0, 2023-07-14T11:29:31.320 INFO:teuthology.orchestra.run.smithi045.stdout: "expos": 4194304, 2023-07-14T11:29:31.320 INFO:teuthology.orchestra.run.smithi045.stdout: "wrpos": 0, 2023-07-14T11:29:31.320 INFO:teuthology.orchestra.run.smithi045.stdout: "rdpos": 7149730, 2023-07-14T11:29:31.321 INFO:teuthology.orchestra.run.smithi045.stdout: "jlat": { 2023-07-14T11:29:31.321 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.321 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.321 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.321 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.322 INFO:teuthology.orchestra.run.smithi045.stdout: "replayed": 1608 2023-07-14T11:29:31.322 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.322 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_mem": { 2023-07-14T11:29:31.322 INFO:teuthology.orchestra.run.smithi045.stdout: "ino": 796, 2023-07-14T11:29:31.322 INFO:teuthology.orchestra.run.smithi045.stdout: "ino+": 957, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "ino-": 161, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "dir": 51, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "dir+": 58, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "dir-": 7, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "dn": 853, 2023-07-14T11:29:31.323 INFO:teuthology.orchestra.run.smithi045.stdout: "dn+": 1040, 2023-07-14T11:29:31.324 INFO:teuthology.orchestra.run.smithi045.stdout: "dn-": 187, 2023-07-14T11:29:31.324 INFO:teuthology.orchestra.run.smithi045.stdout: "cap": 0, 2023-07-14T11:29:31.324 INFO:teuthology.orchestra.run.smithi045.stdout: "cap+": 0, 2023-07-14T11:29:31.325 INFO:teuthology.orchestra.run.smithi045.stdout: "cap-": 0, 2023-07-14T11:29:31.325 INFO:teuthology.orchestra.run.smithi045.stdout: "rss": 49896, 2023-07-14T11:29:31.325 INFO:teuthology.orchestra.run.smithi045.stdout: "heap": 207156 2023-07-14T11:29:31.326 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.326 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_server": { 2023-07-14T11:29:31.326 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_client_request": 0, 2023-07-14T11:29:31.327 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_server_request": 0, 2023-07-14T11:29:31.327 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_request": 0, 2023-07-14T11:29:31.327 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_session": 0, 2023-07-14T11:29:31.327 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_peer_request": 0, 2023-07-14T11:29:31.328 INFO:teuthology.orchestra.run.smithi045.stdout: "req_create_latency": { 2023-07-14T11:29:31.328 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.328 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.328 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.328 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.329 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getattr_latency": { 2023-07-14T11:29:31.329 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.329 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.329 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.330 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.330 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:31.330 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.330 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.330 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.331 INFO:teuthology.orchestra.run.smithi045.stdout: "req_link_latency": { 2023-07-14T11:29:31.331 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.331 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.332 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.332 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookup_latency": { 2023-07-14T11:29:31.332 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.333 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.333 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.333 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.333 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:31.334 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.334 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.334 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.334 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.335 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupino_latency": { 2023-07-14T11:29:31.335 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.335 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.335 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.336 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.336 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupname_latency": { 2023-07-14T11:29:31.336 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.336 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.337 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.337 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.337 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:31.337 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.338 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.338 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.338 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.338 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:31.339 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.339 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.339 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.340 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.340 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lssnap_latency": { 2023-07-14T11:29:31.340 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.340 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.341 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.341 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.341 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mkdir_latency": { 2023-07-14T11:29:31.341 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.342 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.342 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.342 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.343 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mknod_latency": { 2023-07-14T11:29:31.343 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.343 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.343 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.343 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.344 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mksnap_latency": { 2023-07-14T11:29:31.344 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.344 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.344 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.344 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.345 INFO:teuthology.orchestra.run.smithi045.stdout: "req_open_latency": { 2023-07-14T11:29:31.345 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.345 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.345 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.346 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.346 INFO:teuthology.orchestra.run.smithi045.stdout: "req_readdir_latency": { 2023-07-14T11:29:31.347 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.347 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.347 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.348 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.348 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rename_latency": { 2023-07-14T11:29:31.348 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.348 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.349 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.349 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.349 INFO:teuthology.orchestra.run.smithi045.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:31.349 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.349 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.350 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.350 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.350 INFO:teuthology.orchestra.run.smithi045.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:31.350 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.351 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.351 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.351 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.351 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmdir_latency": { 2023-07-14T11:29:31.351 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.352 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.352 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.352 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.353 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:31.353 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.353 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.354 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.354 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.354 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:31.355 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.355 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.355 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.355 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.355 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setattr_latency": { 2023-07-14T11:29:31.356 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.356 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.356 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.356 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.356 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:31.357 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.357 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.357 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.358 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:31.358 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.358 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.359 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.359 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setlayout_latency": { 2023-07-14T11:29:31.359 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.359 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.360 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.360 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.360 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setxattr_latency": { 2023-07-14T11:29:31.360 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.361 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.361 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.361 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.361 INFO:teuthology.orchestra.run.smithi045.stdout: "req_symlink_latency": { 2023-07-14T11:29:31.361 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.362 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.362 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.362 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.362 INFO:teuthology.orchestra.run.smithi045.stdout: "req_unlink_latency": { 2023-07-14T11:29:31.362 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.363 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.363 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.363 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.364 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:31.364 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:31.364 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:31.364 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.364 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.365 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.365 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.365 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.365 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_sessions": { 2023-07-14T11:29:31.365 INFO:teuthology.orchestra.run.smithi045.stdout: "session_count": 2, 2023-07-14T11:29:31.366 INFO:teuthology.orchestra.run.smithi045.stdout: "session_add": 3, 2023-07-14T11:29:31.366 INFO:teuthology.orchestra.run.smithi045.stdout: "session_remove": 1, 2023-07-14T11:29:31.366 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_open": 3, 2023-07-14T11:29:31.366 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_stale": 0, 2023-07-14T11:29:31.366 INFO:teuthology.orchestra.run.smithi045.stdout: "total_load": 0, 2023-07-14T11:29:31.367 INFO:teuthology.orchestra.run.smithi045.stdout: "average_load": 0, 2023-07-14T11:29:31.367 INFO:teuthology.orchestra.run.smithi045.stdout: "avg_session_uptime": 4234 2023-07-14T11:29:31.367 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.367 INFO:teuthology.orchestra.run.smithi045.stdout: "mempool": { 2023-07-14T11:29:31.368 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:31.368 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:31.368 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:31.369 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:31.369 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:31.369 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:31.369 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:31.370 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:31.370 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:31.370 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:31.370 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:31.370 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:31.371 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:31.371 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:31.371 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:31.371 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:31.371 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:31.372 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:31.372 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:31.372 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:31.372 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:31.373 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:31.373 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:31.373 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:31.373 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:31.374 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:31.374 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:31.374 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:31.374 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:31.374 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:31.375 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:31.375 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_items": 0, 2023-07-14T11:29:31.375 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:31.375 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:31.375 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:31.376 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:31.376 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_bytes": 222089, 2023-07-14T11:29:31.376 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_items": 916, 2023-07-14T11:29:31.376 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_bytes": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_items": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:31.377 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:31.378 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:31.378 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:31.378 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_items": 183, 2023-07-14T11:29:31.379 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:31.379 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:31.379 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:31.380 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_items": 0, 2023-07-14T11:29:31.380 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_bytes": 2557478, 2023-07-14T11:29:31.380 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_items": 28822, 2023-07-14T11:29:31.380 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:31.380 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_items": 0, 2023-07-14T11:29:31.381 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:31.381 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_items": 0 2023-07-14T11:29:31.381 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.382 INFO:teuthology.orchestra.run.smithi045.stdout: "objecter": { 2023-07-14T11:29:31.382 INFO:teuthology.orchestra.run.smithi045.stdout: "op_active": 0, 2023-07-14T11:29:31.383 INFO:teuthology.orchestra.run.smithi045.stdout: "op_laggy": 0, 2023-07-14T11:29:31.383 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send": 3461, 2023-07-14T11:29:31.383 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send_bytes": 21, 2023-07-14T11:29:31.383 INFO:teuthology.orchestra.run.smithi045.stdout: "op_resend": 3, 2023-07-14T11:29:31.384 INFO:teuthology.orchestra.run.smithi045.stdout: "op_reply": 3461, 2023-07-14T11:29:31.384 INFO:teuthology.orchestra.run.smithi045.stdout: "op_latency": { 2023-07-14T11:29:31.384 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 3461, 2023-07-14T11:29:31.385 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 2.074965528, 2023-07-14T11:29:31.385 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000599527 2023-07-14T11:29:31.385 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.385 INFO:teuthology.orchestra.run.smithi045.stdout: "op_inflight": 0, 2023-07-14T11:29:31.385 INFO:teuthology.orchestra.run.smithi045.stdout: "oplen_avg": { 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 3461, 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 3462 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: "op": 3461, 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: "op_r": 3461, 2023-07-14T11:29:31.386 INFO:teuthology.orchestra.run.smithi045.stdout: "op_w": 0, 2023-07-14T11:29:31.387 INFO:teuthology.orchestra.run.smithi045.stdout: "op_rmw": 0, 2023-07-14T11:29:31.387 INFO:teuthology.orchestra.run.smithi045.stdout: "op_pg": 0, 2023-07-14T11:29:31.387 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_stat": 2192, 2023-07-14T11:29:31.387 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_create": 0, 2023-07-14T11:29:31.387 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_read": 1267, 2023-07-14T11:29:31.388 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_write": 0, 2023-07-14T11:29:31.388 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writefull": 0, 2023-07-14T11:29:31.388 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writesame": 0, 2023-07-14T11:29:31.388 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_append": 0, 2023-07-14T11:29:31.389 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_zero": 0, 2023-07-14T11:29:31.389 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_truncate": 0, 2023-07-14T11:29:31.389 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_delete": 0, 2023-07-14T11:29:31.390 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_mapext": 0, 2023-07-14T11:29:31.390 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:31.390 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:31.390 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_getxattr": 1, 2023-07-14T11:29:31.391 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_setxattr": 0, 2023-07-14T11:29:31.391 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:31.391 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:31.391 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:31.391 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_call": 0, 2023-07-14T11:29:31.392 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_watch": 0, 2023-07-14T11:29:31.392 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_notify": 0, 2023-07-14T11:29:31.392 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:31.392 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls": 0, 2023-07-14T11:29:31.393 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:31.393 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_other": 0, 2023-07-14T11:29:31.393 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_active": 0, 2023-07-14T11:29:31.393 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_send": 0, 2023-07-14T11:29:31.394 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_resend": 0, 2023-07-14T11:29:31.394 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_ping": 0, 2023-07-14T11:29:31.394 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_active": 0, 2023-07-14T11:29:31.394 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_send": 0, 2023-07-14T11:29:31.395 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_resend": 0, 2023-07-14T11:29:31.395 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_active": 0, 2023-07-14T11:29:31.395 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_send": 0, 2023-07-14T11:29:31.395 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_resend": 0, 2023-07-14T11:29:31.395 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_active": 0, 2023-07-14T11:29:31.396 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_send": 0, 2023-07-14T11:29:31.396 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_resend": 0, 2023-07-14T11:29:31.396 INFO:teuthology.orchestra.run.smithi045.stdout: "command_active": 0, 2023-07-14T11:29:31.396 INFO:teuthology.orchestra.run.smithi045.stdout: "command_send": 0, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "command_resend": 0, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "map_epoch": 115, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "map_full": 0, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "map_inc": 25, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_sessions": 5, 2023-07-14T11:29:31.397 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_open": 6, 2023-07-14T11:29:31.398 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_close": 1, 2023-07-14T11:29:31.398 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_laggy": 0, 2023-07-14T11:29:31.398 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_wr": 0, 2023-07-14T11:29:31.398 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_rd": 2, 2023-07-14T11:29:31.398 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_del": 0 2023-07-14T11:29:31.399 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.399 INFO:teuthology.orchestra.run.smithi045.stdout: "oft": { 2023-07-14T11:29:31.399 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_objs": 0, 2023-07-14T11:29:31.400 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:31.400 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_updates": 0, 2023-07-14T11:29:31.400 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_removes": 0 2023-07-14T11:29:31.401 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.401 INFO:teuthology.orchestra.run.smithi045.stdout: "osd-slow-ops": { 2023-07-14T11:29:31.401 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_ops_count": 0 2023-07-14T11:29:31.401 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.401 INFO:teuthology.orchestra.run.smithi045.stdout: "purge_queue": { 2023-07-14T11:29:31.402 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:31.402 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:31.402 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing": 0, 2023-07-14T11:29:31.402 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:31.402 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executed": 0, 2023-07-14T11:29:31.403 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:31.403 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.403 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:31.403 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:31.403 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:31.404 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:31.404 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 4049, 2023-07-14T11:29:31.404 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 6259087, 2023-07-14T11:29:31.404 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:31.404 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 4049, 2023-07-14T11:29:31.405 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:31.405 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:31.405 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 4049, 2023-07-14T11:29:31.406 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 6259087, 2023-07-14T11:29:31.406 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:31.406 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.406 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.407 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.407 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.407 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.407 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:31.407 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:31.408 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:31.408 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:31.408 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:31.408 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:31.408 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:31.409 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:31.409 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 3461, 2023-07-14T11:29:31.409 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 2956005, 2023-07-14T11:29:31.409 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 3461, 2023-07-14T11:29:31.410 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 2956005, 2023-07-14T11:29:31.410 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:31.410 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.411 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.411 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.411 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.411 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.411 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:31.412 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:31.412 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 1024, 2023-07-14T11:29:31.412 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:31.412 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:31.413 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:31.413 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:31.413 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:31.413 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 3461, 2023-07-14T11:29:31.413 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 3461, 2023-07-14T11:29:31.414 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 3461, 2023-07-14T11:29:31.414 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 3461, 2023-07-14T11:29:31.414 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:31.414 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.415 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.415 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.415 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.415 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.415 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:31.416 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:31.416 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:31.416 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:31.416 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:31.417 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:31.417 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:31.417 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:31.417 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:31.417 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:31.418 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 0, 2023-07-14T11:29:31.418 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 0, 2023-07-14T11:29:31.418 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:31.418 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.418 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.419 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.419 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.419 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:31.419 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle-0x5567509cc460": { 2023-07-14T11:29:31.420 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:31.420 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:31.420 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:31.420 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:31.421 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:31.421 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:31.421 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:31.421 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:31.422 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:31.422 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 0, 2023-07-14T11:29:31.422 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 0, 2023-07-14T11:29:31.422 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:31.422 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:31.423 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:31.423 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:31.423 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.423 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:31.423 INFO:teuthology.orchestra.run.smithi045.stdout:} 2023-07-14T11:29:32.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:31 smithi130 ceph-mon[129405]: pgmap v546: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:32.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:31 smithi040 ceph-mon[97954]: pgmap v546: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:32.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:31 smithi045 ceph-mon[128791]: pgmap v546: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:33.626 INFO:tasks.check_counter:Daemon mds.e mds.exported=0 2023-07-14T11:29:33.626 INFO:tasks.check_counter:Daemon mds.e mds.imported=0 2023-07-14T11:29:33.627 INFO:tasks.check_counter:Daemon mds.e mds.root_rsnaps=0 2023-07-14T11:29:33.627 INFO:tasks.check_counter:Daemon mds.e mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:33.627 INFO:tasks.check_counter:Daemon mds.e mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:33.627 DEBUG:tasks.check_counter:Getting stats from h 2023-07-14T11:29:33.628 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.h perf dump 2023-07-14T11:29:34.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:33 smithi130 ceph-mon[129405]: pgmap v547: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:34.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:33 smithi040 ceph-mon[97954]: pgmap v547: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:34.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:33 smithi045 ceph-mon[128791]: pgmap v547: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:36.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:35 smithi130 ceph-mon[129405]: pgmap v548: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:36.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:35 smithi040 ceph-mon[97954]: pgmap v548: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:36.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:35 smithi045 ceph-mon[128791]: pgmap v548: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:38.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:37 smithi130 ceph-mon[129405]: pgmap v549: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:38.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:37 smithi040 ceph-mon[97954]: pgmap v549: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:38.357 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:37 smithi045 ceph-mon[128791]: pgmap v549: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:39.567 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:29:40.219 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:39 smithi045 ceph-mon[128791]: pgmap v550: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:40.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:39 smithi130 ceph-mon[129405]: pgmap v550: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:40.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:39 smithi040 ceph-mon[97954]: pgmap v550: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:41.420 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2023-07-14T11:29:41.420 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:41.420 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 1364, 2023-07-14T11:29:41.421 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 1342, 2023-07-14T11:29:41.421 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 3227261, 2023-07-14T11:29:41.421 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 3254022, 2023-07-14T11:29:41.421 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 18, 2023-07-14T11:29:41.421 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:41.422 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.123581846, 2023-07-14T11:29:41.422 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.041104114, 2023-07-14T11:29:41.422 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.070074788, 2023-07-14T11:29:41.422 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.023100320, 2023-07-14T11:29:41.422 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:41.423 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1341, 2023-07-14T11:29:41.423 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.110334901, 2023-07-14T11:29:41.423 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000082278 2023-07-14T11:29:41.423 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.424 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:41.424 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 217, 2023-07-14T11:29:41.424 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000041742, 2023-07-14T11:29:41.424 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000192 2023-07-14T11:29:41.425 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.425 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 2744768, 2023-07-14T11:29:41.425 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 657776 2023-07-14T11:29:41.425 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.425 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:41.426 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 2219, 2023-07-14T11:29:41.426 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 2781, 2023-07-14T11:29:41.426 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 1199893, 2023-07-14T11:29:41.426 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 2303635, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 18, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 6, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.114671964, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.045433695, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.152251550, 2023-07-14T11:29:41.427 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.006531303, 2023-07-14T11:29:41.428 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:41.428 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 2778, 2023-07-14T11:29:41.428 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.715943783, 2023-07-14T11:29:41.428 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000257719 2023-07-14T11:29:41.429 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.429 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:41.429 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 3178, 2023-07-14T11:29:41.429 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000689660, 2023-07-14T11:29:41.429 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000217 2023-07-14T11:29:41.430 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.430 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 1072, 2023-07-14T11:29:41.430 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 123872 2023-07-14T11:29:41.430 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.431 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:41.431 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 4464105, 2023-07-14T11:29:41.431 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 4462240, 2023-07-14T11:29:41.431 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 742099435, 2023-07-14T11:29:41.431 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 708425313, 2023-07-14T11:29:41.432 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 10, 2023-07-14T11:29:41.432 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 7, 2023-07-14T11:29:41.432 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 217.956089625, 2023-07-14T11:29:41.432 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 92.557669433, 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 155.633479398, 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.009340892, 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 4462239, 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 49.032287831, 2023-07-14T11:29:41.433 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000010988 2023-07-14T11:29:41.434 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.434 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:41.434 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 8925056, 2023-07-14T11:29:41.434 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 1.116842760, 2023-07-14T11:29:41.435 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000125 2023-07-14T11:29:41.435 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.435 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 7120, 2023-07-14T11:29:41.435 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 20096 2023-07-14T11:29:41.435 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.436 INFO:teuthology.orchestra.run.smithi045.stdout: "cct": { 2023-07-14T11:29:41.436 INFO:teuthology.orchestra.run.smithi045.stdout: "total_workers": 1, 2023-07-14T11:29:41.436 INFO:teuthology.orchestra.run.smithi045.stdout: "unhealthy_workers": 0 2023-07-14T11:29:41.436 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.436 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-MDSRank": { 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 913, 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.824950307, 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000903560 2023-07-14T11:29:41.437 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.438 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.438 INFO:teuthology.orchestra.run.smithi045.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:41.438 INFO:teuthology.orchestra.run.smithi045.stdout: "queue_len": 0, 2023-07-14T11:29:41.438 INFO:teuthology.orchestra.run.smithi045.stdout: "complete_latency": { 2023-07-14T11:29:41.438 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 2, 2023-07-14T11:29:41.439 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000006698, 2023-07-14T11:29:41.439 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000003349 2023-07-14T11:29:41.439 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.439 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.439 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2023-07-14T11:29:41.440 INFO:teuthology.orchestra.run.smithi045.stdout: "request": 245, 2023-07-14T11:29:41.440 INFO:teuthology.orchestra.run.smithi045.stdout: "reply": 235, 2023-07-14T11:29:41.440 INFO:teuthology.orchestra.run.smithi045.stdout: "reply_latency": { 2023-07-14T11:29:41.440 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 235, 2023-07-14T11:29:41.440 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 287.791331530, 2023-07-14T11:29:41.441 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 1.224643963 2023-07-14T11:29:41.441 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.441 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_reply": 0, 2023-07-14T11:29:41.441 INFO:teuthology.orchestra.run.smithi045.stdout: "forward": 10, 2023-07-14T11:29:41.441 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_complete": 1, 2023-07-14T11:29:41.442 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:41.442 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_commit": 59, 2023-07-14T11:29:41.442 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_split": 6, 2023-07-14T11:29:41.442 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_merge": 0, 2023-07-14T11:29:41.442 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes": 607, 2023-07-14T11:29:41.443 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_top": 260, 2023-07-14T11:29:41.443 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_bottom": 333, 2023-07-14T11:29:41.443 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pin_tail": 14, 2023-07-14T11:29:41.443 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_pinned": 235, 2023-07-14T11:29:41.444 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_expired": 298, 2023-07-14T11:29:41.444 INFO:teuthology.orchestra.run.smithi045.stdout: "inodes_with_caps": 22, 2023-07-14T11:29:41.444 INFO:teuthology.orchestra.run.smithi045.stdout: "caps": 22, 2023-07-14T11:29:41.444 INFO:teuthology.orchestra.run.smithi045.stdout: "subtrees": 50, 2023-07-14T11:29:41.445 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse": 2090, 2023-07-14T11:29:41.445 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_hit": 1789, 2023-07-14T11:29:41.445 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_forward": 0, 2023-07-14T11:29:41.445 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_discover": 5, 2023-07-14T11:29:41.445 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:41.446 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:41.446 INFO:teuthology.orchestra.run.smithi045.stdout: "traverse_lock": 275, 2023-07-14T11:29:41.446 INFO:teuthology.orchestra.run.smithi045.stdout: "load_cent": 0, 2023-07-14T11:29:41.447 INFO:teuthology.orchestra.run.smithi045.stdout: "q": 0, 2023-07-14T11:29:41.447 INFO:teuthology.orchestra.run.smithi045.stdout: "exported": 11, 2023-07-14T11:29:41.447 INFO:teuthology.orchestra.run.smithi045.stdout: "exported_inodes": 263, 2023-07-14T11:29:41.447 INFO:teuthology.orchestra.run.smithi045.stdout: "imported": 8, 2023-07-14T11:29:41.447 INFO:teuthology.orchestra.run.smithi045.stdout: "imported_inodes": 369, 2023-07-14T11:29:41.448 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:41.448 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_backtrace_fetch": 485, 2023-07-14T11:29:41.448 INFO:teuthology.orchestra.run.smithi045.stdout: "openino_peer_discover": 70, 2023-07-14T11:29:41.448 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rfiles": 1028, 2023-07-14T11:29:41.448 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rbytes": 5852694522, 2023-07-14T11:29:41.449 INFO:teuthology.orchestra.run.smithi045.stdout: "root_rsnaps": 4, 2023-07-14T11:29:41.449 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_fetch": 485, 2023-07-14T11:29:41.449 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:41.449 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:41.449 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:41.450 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:41.450 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:41.450 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:41.450 INFO:teuthology.orchestra.run.smithi045.stdout: "scrub_file_inodes": 485, 2023-07-14T11:29:41.450 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_inode_file_caps": 8, 2023-07-14T11:29:41.451 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_revoke": 595, 2023-07-14T11:29:41.451 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_grant": 928, 2023-07-14T11:29:41.451 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:41.451 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:41.451 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:41.452 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps": 1423, 2023-07-14T11:29:41.452 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_caps_dirty": 28, 2023-07-14T11:29:41.452 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_cap_release": 16, 2023-07-14T11:29:41.452 INFO:teuthology.orchestra.run.smithi045.stdout: "process_request_cap_release": 155 2023-07-14T11:29:41.452 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.453 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_cache": { 2023-07-14T11:29:41.453 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update": 0, 2023-07-14T11:29:41.453 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:41.453 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_try_discover": 0, 2023-07-14T11:29:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_send_discover": 45, 2023-07-14T11:29:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "dir_handle_discover": 6, 2023-07-14T11:29:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays": 0, 2023-07-14T11:29:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_created": 0, 2023-07-14T11:29:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_enqueued": 0, 2023-07-14T11:29:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_reintegrated": 42, 2023-07-14T11:29:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: "strays_migrated": 0, 2023-07-14T11:29:41.456 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:41.456 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:41.456 INFO:teuthology.orchestra.run.smithi045.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:41.457 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_started": 0, 2023-07-14T11:29:41.457 INFO:teuthology.orchestra.run.smithi045.stdout: "recovery_completed": 0, 2023-07-14T11:29:41.458 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:41.458 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_exportdir": 13, 2023-07-14T11:29:41.458 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_flush": 0, 2023-07-14T11:29:41.459 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragmentdir": 6, 2023-07-14T11:29:41.459 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:41.459 INFO:teuthology.orchestra.run.smithi045.stdout: "ireq_inodestats": 0 2023-07-14T11:29:41.460 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.460 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_log": { 2023-07-14T11:29:41.461 INFO:teuthology.orchestra.run.smithi045.stdout: "evadd": 1746, 2023-07-14T11:29:41.461 INFO:teuthology.orchestra.run.smithi045.stdout: "evex": 0, 2023-07-14T11:29:41.461 INFO:teuthology.orchestra.run.smithi045.stdout: "evtrm": 0, 2023-07-14T11:29:41.462 INFO:teuthology.orchestra.run.smithi045.stdout: "ev": 1746, 2023-07-14T11:29:41.462 INFO:teuthology.orchestra.run.smithi045.stdout: "evexg": 0, 2023-07-14T11:29:41.463 INFO:teuthology.orchestra.run.smithi045.stdout: "evexd": 0, 2023-07-14T11:29:41.463 INFO:teuthology.orchestra.run.smithi045.stdout: "segadd": 2, 2023-07-14T11:29:41.463 INFO:teuthology.orchestra.run.smithi045.stdout: "segex": 0, 2023-07-14T11:29:41.464 INFO:teuthology.orchestra.run.smithi045.stdout: "segtrm": 0, 2023-07-14T11:29:41.464 INFO:teuthology.orchestra.run.smithi045.stdout: "seg": 2, 2023-07-14T11:29:41.464 INFO:teuthology.orchestra.run.smithi045.stdout: "segexg": 0, 2023-07-14T11:29:41.465 INFO:teuthology.orchestra.run.smithi045.stdout: "segexd": 0, 2023-07-14T11:29:41.465 INFO:teuthology.orchestra.run.smithi045.stdout: "expos": 4194304, 2023-07-14T11:29:41.466 INFO:teuthology.orchestra.run.smithi045.stdout: "wrpos": 6912437, 2023-07-14T11:29:41.466 INFO:teuthology.orchestra.run.smithi045.stdout: "rdpos": 0, 2023-07-14T11:29:41.466 INFO:teuthology.orchestra.run.smithi045.stdout: "jlat": { 2023-07-14T11:29:41.467 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 395, 2023-07-14T11:29:41.467 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 2.788267406, 2023-07-14T11:29:41.468 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.007058904 2023-07-14T11:29:41.468 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.468 INFO:teuthology.orchestra.run.smithi045.stdout: "replayed": 0 2023-07-14T11:29:41.469 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.469 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_mem": { 2023-07-14T11:29:41.470 INFO:teuthology.orchestra.run.smithi045.stdout: "ino": 584, 2023-07-14T11:29:41.470 INFO:teuthology.orchestra.run.smithi045.stdout: "ino+": 876, 2023-07-14T11:29:41.470 INFO:teuthology.orchestra.run.smithi045.stdout: "ino-": 292, 2023-07-14T11:29:41.471 INFO:teuthology.orchestra.run.smithi045.stdout: "dir": 68, 2023-07-14T11:29:41.471 INFO:teuthology.orchestra.run.smithi045.stdout: "dir+": 75, 2023-07-14T11:29:41.471 INFO:teuthology.orchestra.run.smithi045.stdout: "dir-": 7, 2023-07-14T11:29:41.472 INFO:teuthology.orchestra.run.smithi045.stdout: "dn": 607, 2023-07-14T11:29:41.472 INFO:teuthology.orchestra.run.smithi045.stdout: "dn+": 908, 2023-07-14T11:29:41.473 INFO:teuthology.orchestra.run.smithi045.stdout: "dn-": 301, 2023-07-14T11:29:41.473 INFO:teuthology.orchestra.run.smithi045.stdout: "cap": 22, 2023-07-14T11:29:41.473 INFO:teuthology.orchestra.run.smithi045.stdout: "cap+": 842, 2023-07-14T11:29:41.474 INFO:teuthology.orchestra.run.smithi045.stdout: "cap-": 820, 2023-07-14T11:29:41.474 INFO:teuthology.orchestra.run.smithi045.stdout: "rss": 63444, 2023-07-14T11:29:41.475 INFO:teuthology.orchestra.run.smithi045.stdout: "heap": 198964 2023-07-14T11:29:41.475 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.475 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_server": { 2023-07-14T11:29:41.476 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_client_request": 1337, 2023-07-14T11:29:41.476 INFO:teuthology.orchestra.run.smithi045.stdout: "dispatch_server_request": 4, 2023-07-14T11:29:41.477 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_request": 245, 2023-07-14T11:29:41.477 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_client_session": 40, 2023-07-14T11:29:41.478 INFO:teuthology.orchestra.run.smithi045.stdout: "handle_peer_request": 300, 2023-07-14T11:29:41.478 INFO:teuthology.orchestra.run.smithi045.stdout: "req_create_latency": { 2023-07-14T11:29:41.478 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 2, 2023-07-14T11:29:41.479 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.006570861, 2023-07-14T11:29:41.479 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.003285430 2023-07-14T11:29:41.480 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.480 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getattr_latency": { 2023-07-14T11:29:41.481 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 3, 2023-07-14T11:29:41.481 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000981897, 2023-07-14T11:29:41.481 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000327299 2023-07-14T11:29:41.482 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.482 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:41.483 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.483 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.483 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.484 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.484 INFO:teuthology.orchestra.run.smithi045.stdout: "req_link_latency": { 2023-07-14T11:29:41.485 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.485 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.486 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.486 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.486 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookup_latency": { 2023-07-14T11:29:41.487 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 26, 2023-07-14T11:29:41.487 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.534273129, 2023-07-14T11:29:41.488 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.020548966 2023-07-14T11:29:41.488 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.489 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:41.489 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.489 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.490 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.490 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.491 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupino_latency": { 2023-07-14T11:29:41.491 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.491 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.492 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.492 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.493 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupname_latency": { 2023-07-14T11:29:41.493 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.494 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.494 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.495 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.495 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:41.495 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.496 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.496 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.497 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.497 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:41.497 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.498 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.498 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.499 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.499 INFO:teuthology.orchestra.run.smithi045.stdout: "req_lssnap_latency": { 2023-07-14T11:29:41.499 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.500 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.500 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.500 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.500 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mkdir_latency": { 2023-07-14T11:29:41.501 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.501 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.502 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.502 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.503 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mknod_latency": { 2023-07-14T11:29:41.503 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.504 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.504 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.504 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.505 INFO:teuthology.orchestra.run.smithi045.stdout: "req_mksnap_latency": { 2023-07-14T11:29:41.505 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.506 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.506 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.506 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.506 INFO:teuthology.orchestra.run.smithi045.stdout: "req_open_latency": { 2023-07-14T11:29:41.506 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 7, 2023-07-14T11:29:41.507 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.005947253, 2023-07-14T11:29:41.507 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000849607 2023-07-14T11:29:41.507 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.507 INFO:teuthology.orchestra.run.smithi045.stdout: "req_readdir_latency": { 2023-07-14T11:29:41.507 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 149, 2023-07-14T11:29:41.508 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 54.783740257, 2023-07-14T11:29:41.508 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.367676109 2023-07-14T11:29:41.508 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.508 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rename_latency": { 2023-07-14T11:29:41.509 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 48, 2023-07-14T11:29:41.509 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 232.459818133, 2023-07-14T11:29:41.509 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 4.842912877 2023-07-14T11:29:41.510 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.510 INFO:teuthology.orchestra.run.smithi045.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:41.511 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.511 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.512 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.512 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.512 INFO:teuthology.orchestra.run.smithi045.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:41.513 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.513 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.514 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.514 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.515 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmdir_latency": { 2023-07-14T11:29:41.515 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.516 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.516 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.516 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.517 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:41.517 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.517 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.518 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.518 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.518 INFO:teuthology.orchestra.run.smithi045.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:41.519 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.519 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.520 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.520 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.520 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setattr_latency": { 2023-07-14T11:29:41.520 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.521 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.521 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.521 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.521 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:41.521 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.522 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.522 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.522 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.522 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:41.523 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.523 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.523 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.523 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.523 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setlayout_latency": { 2023-07-14T11:29:41.524 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.524 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.524 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.524 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.525 INFO:teuthology.orchestra.run.smithi045.stdout: "req_setxattr_latency": { 2023-07-14T11:29:41.525 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.525 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.525 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.525 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.526 INFO:teuthology.orchestra.run.smithi045.stdout: "req_symlink_latency": { 2023-07-14T11:29:41.526 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.526 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.526 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.527 INFO:teuthology.orchestra.run.smithi045.stdout: "req_unlink_latency": { 2023-07-14T11:29:41.527 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.527 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.527 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.527 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.528 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:41.528 INFO:teuthology.orchestra.run.smithi045.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:41.528 INFO:teuthology.orchestra.run.smithi045.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:41.528 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.529 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.529 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.529 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.530 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.530 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_sessions": { 2023-07-14T11:29:41.531 INFO:teuthology.orchestra.run.smithi045.stdout: "session_count": 2, 2023-07-14T11:29:41.531 INFO:teuthology.orchestra.run.smithi045.stdout: "session_add": 3, 2023-07-14T11:29:41.532 INFO:teuthology.orchestra.run.smithi045.stdout: "session_remove": 1, 2023-07-14T11:29:41.532 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_open": 2, 2023-07-14T11:29:41.532 INFO:teuthology.orchestra.run.smithi045.stdout: "sessions_stale": 0, 2023-07-14T11:29:41.532 INFO:teuthology.orchestra.run.smithi045.stdout: "total_load": 129, 2023-07-14T11:29:41.532 INFO:teuthology.orchestra.run.smithi045.stdout: "average_load": 64, 2023-07-14T11:29:41.533 INFO:teuthology.orchestra.run.smithi045.stdout: "avg_session_uptime": 1022 2023-07-14T11:29:41.533 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.533 INFO:teuthology.orchestra.run.smithi045.stdout: "mempool": { 2023-07-14T11:29:41.533 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:41.533 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:41.534 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:41.534 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:41.534 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:41.534 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:41.534 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:41.535 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:41.535 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:41.535 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:41.535 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:41.535 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:41.536 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:41.536 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:41.536 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:41.536 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:41.536 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:41.537 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:41.538 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:41.538 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:41.538 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:41.538 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:41.538 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:41.539 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:41.539 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:41.539 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:41.539 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_items": 0, 2023-07-14T11:29:41.539 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:41.540 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:41.540 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:41.540 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:41.540 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_bytes": 366799, 2023-07-14T11:29:41.540 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_items": 403, 2023-07-14T11:29:41.541 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_bytes": 88, 2023-07-14T11:29:41.541 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_items": 1, 2023-07-14T11:29:41.541 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_bytes": 0, 2023-07-14T11:29:41.541 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_items": 0, 2023-07-14T11:29:41.541 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:41.542 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:41.542 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:41.542 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:41.542 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:41.542 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_items": 183, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_items": 0, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_bytes": 1868722, 2023-07-14T11:29:41.543 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_items": 18491, 2023-07-14T11:29:41.544 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:41.544 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_items": 0, 2023-07-14T11:29:41.544 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:41.544 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_items": 0 2023-07-14T11:29:41.544 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.545 INFO:teuthology.orchestra.run.smithi045.stdout: "objecter": { 2023-07-14T11:29:41.545 INFO:teuthology.orchestra.run.smithi045.stdout: "op_active": 0, 2023-07-14T11:29:41.545 INFO:teuthology.orchestra.run.smithi045.stdout: "op_laggy": 0, 2023-07-14T11:29:41.545 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send": 1016, 2023-07-14T11:29:41.545 INFO:teuthology.orchestra.run.smithi045.stdout: "op_send_bytes": 3218910, 2023-07-14T11:29:41.546 INFO:teuthology.orchestra.run.smithi045.stdout: "op_resend": 28, 2023-07-14T11:29:41.546 INFO:teuthology.orchestra.run.smithi045.stdout: "op_reply": 1016, 2023-07-14T11:29:41.546 INFO:teuthology.orchestra.run.smithi045.stdout: "op_latency": { 2023-07-14T11:29:41.546 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1016, 2023-07-14T11:29:41.546 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 2.480958468, 2023-07-14T11:29:41.547 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.002441888 2023-07-14T11:29:41.547 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.547 INFO:teuthology.orchestra.run.smithi045.stdout: "op_inflight": 0, 2023-07-14T11:29:41.547 INFO:teuthology.orchestra.run.smithi045.stdout: "oplen_avg": { 2023-07-14T11:29:41.547 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 1016, 2023-07-14T11:29:41.548 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 1123 2023-07-14T11:29:41.548 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.548 INFO:teuthology.orchestra.run.smithi045.stdout: "op": 1016, 2023-07-14T11:29:41.548 INFO:teuthology.orchestra.run.smithi045.stdout: "op_r": 486, 2023-07-14T11:29:41.548 INFO:teuthology.orchestra.run.smithi045.stdout: "op_w": 530, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "op_rmw": 0, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "op_pg": 0, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_stat": 0, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_create": 10, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_read": 0, 2023-07-14T11:29:41.549 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_write": 395, 2023-07-14T11:29:41.550 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writefull": 24, 2023-07-14T11:29:41.550 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_writesame": 0, 2023-07-14T11:29:41.550 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_append": 0, 2023-07-14T11:29:41.550 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_zero": 0, 2023-07-14T11:29:41.550 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_truncate": 0, 2023-07-14T11:29:41.551 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_delete": 12, 2023-07-14T11:29:41.551 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_mapext": 0, 2023-07-14T11:29:41.551 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:41.551 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:41.551 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_getxattr": 485, 2023-07-14T11:29:41.552 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_setxattr": 20, 2023-07-14T11:29:41.552 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:41.552 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:41.552 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:41.552 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_call": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_watch": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_notify": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:41.553 INFO:teuthology.orchestra.run.smithi045.stdout: "osdop_other": 0, 2023-07-14T11:29:41.554 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_active": 0, 2023-07-14T11:29:41.554 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_send": 0, 2023-07-14T11:29:41.554 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_resend": 0, 2023-07-14T11:29:41.554 INFO:teuthology.orchestra.run.smithi045.stdout: "linger_ping": 0, 2023-07-14T11:29:41.554 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_active": 0, 2023-07-14T11:29:41.555 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_send": 0, 2023-07-14T11:29:41.555 INFO:teuthology.orchestra.run.smithi045.stdout: "poolop_resend": 0, 2023-07-14T11:29:41.555 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_active": 0, 2023-07-14T11:29:41.555 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_send": 0, 2023-07-14T11:29:41.555 INFO:teuthology.orchestra.run.smithi045.stdout: "poolstat_resend": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_active": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_send": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "statfs_resend": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "command_active": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "command_send": 0, 2023-07-14T11:29:41.556 INFO:teuthology.orchestra.run.smithi045.stdout: "command_resend": 0, 2023-07-14T11:29:41.557 INFO:teuthology.orchestra.run.smithi045.stdout: "map_epoch": 115, 2023-07-14T11:29:41.557 INFO:teuthology.orchestra.run.smithi045.stdout: "map_full": 0, 2023-07-14T11:29:41.557 INFO:teuthology.orchestra.run.smithi045.stdout: "map_inc": 25, 2023-07-14T11:29:41.557 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_sessions": 12, 2023-07-14T11:29:41.557 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_open": 12, 2023-07-14T11:29:41.558 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_session_close": 0, 2023-07-14T11:29:41.558 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_laggy": 0, 2023-07-14T11:29:41.558 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_wr": 158, 2023-07-14T11:29:41.558 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_rd": 2, 2023-07-14T11:29:41.558 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_del": 17 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: "oft": { 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_objs": 1, 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_updates": 0, 2023-07-14T11:29:41.559 INFO:teuthology.orchestra.run.smithi045.stdout: "omap_total_removes": 0 2023-07-14T11:29:41.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.560 INFO:teuthology.orchestra.run.smithi045.stdout: "osd-slow-ops": { 2023-07-14T11:29:41.560 INFO:teuthology.orchestra.run.smithi045.stdout: "slow_ops_count": 268 2023-07-14T11:29:41.560 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.560 INFO:teuthology.orchestra.run.smithi045.stdout: "purge_queue": { 2023-07-14T11:29:41.561 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:41.561 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:41.561 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing": 0, 2023-07-14T11:29:41.561 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:41.561 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_executed": 0, 2023-07-14T11:29:41.562 INFO:teuthology.orchestra.run.smithi045.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:41.562 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.562 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:41.562 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 76, 2023-07-14T11:29:41.562 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:41.563 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:41.563 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 4467686, 2023-07-14T11:29:41.563 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 344400191, 2023-07-14T11:29:41.563 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:41.563 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 4467686, 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 4467685, 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 344400115, 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:41.564 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.565 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.565 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.565 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.565 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.565 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:41.566 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:41.566 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:41.566 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:41.566 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:41.566 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 1016, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 3218894, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 1016, 2023-07-14T11:29:41.567 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 3218894, 2023-07-14T11:29:41.568 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:41.568 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.568 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.568 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.568 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.569 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.569 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:41.569 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:41.569 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 1024, 2023-07-14T11:29:41.569 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:41.570 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:41.570 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:41.570 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:41.570 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:41.570 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 1016, 2023-07-14T11:29:41.571 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 1016, 2023-07-14T11:29:41.571 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 1016, 2023-07-14T11:29:41.571 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 1016, 2023-07-14T11:29:41.571 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:41.571 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.572 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.572 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.572 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.572 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.572 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 0, 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 0, 2023-07-14T11:29:41.573 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:41.574 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:41.574 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:41.574 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:41.574 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 0, 2023-07-14T11:29:41.574 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 0, 2023-07-14T11:29:41.575 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:41.575 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.575 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.575 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.575 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.576 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:41.576 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-write_buf_throttle-0x55c6342f40a0": { 2023-07-14T11:29:41.576 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:41.576 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 3758096384, 2023-07-14T11:29:41.576 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 1746, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 2718133, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 1746, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:41.577 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:41.578 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 395, 2023-07-14T11:29:41.578 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 2718133, 2023-07-14T11:29:41.578 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:41.578 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:41.578 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:41.579 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:41.579 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.579 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:41.579 INFO:teuthology.orchestra.run.smithi045.stdout:} 2023-07-14T11:29:41.799 INFO:tasks.check_counter:Daemon mds.h mds.exported=11 2023-07-14T11:29:41.799 INFO:tasks.check_counter:Daemon mds.h mds.imported=8 2023-07-14T11:29:41.800 INFO:tasks.check_counter:Daemon mds.h mds.root_rsnaps=4 2023-07-14T11:29:41.800 INFO:tasks.check_counter:Daemon mds.h mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:41.800 INFO:tasks.check_counter:Daemon mds.h mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:41.800 DEBUG:tasks.check_counter:Getting stats from k 2023-07-14T11:29:41.801 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.k perf dump 2023-07-14T11:29:42.065 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:41 smithi045 ceph-mon[128791]: pgmap v551: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:42.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:41 smithi130 ceph-mon[129405]: pgmap v551: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:42.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:41 smithi040 ceph-mon[97954]: pgmap v551: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:42.521 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.b/config 2023-07-14T11:29:44.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:43 smithi130 ceph-mon[129405]: pgmap v552: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:44.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:43 smithi040 ceph-mon[97954]: pgmap v552: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:44.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:43 smithi045 ceph-mon[128791]: pgmap v552: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:45.271 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2023-07-14T11:29:45.272 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:45.272 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 0, 2023-07-14T11:29:45.272 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 2, 2023-07-14T11:29:45.273 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 0, 2023-07-14T11:29:45.273 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 0, 2023-07-14T11:29:45.273 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 4, 2023-07-14T11:29:45.273 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 0, 2023-07-14T11:29:45.273 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.001629827, 2023-07-14T11:29:45.274 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.000000000, 2023-07-14T11:29:45.274 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.000362747, 2023-07-14T11:29:45.274 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2023-07-14T11:29:45.274 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:45.275 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:45.275 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:45.275 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:45.275 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.275 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:45.276 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:45.276 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:45.276 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:45.276 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:45.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:45.277 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.277 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:45.277 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 6, 2023-07-14T11:29:45.278 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 253, 2023-07-14T11:29:45.278 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 3264, 2023-07-14T11:29:45.278 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 195936, 2023-07-14T11:29:45.279 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:45.279 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:45.279 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.012242029, 2023-07-14T11:29:45.280 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.007730519, 2023-07-14T11:29:45.280 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.002049876, 2023-07-14T11:29:45.281 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.000000000, 2023-07-14T11:29:45.281 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:45.281 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 251, 2023-07-14T11:29:45.282 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.020598213, 2023-07-14T11:29:45.282 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000082064 2023-07-14T11:29:45.283 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.283 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:45.283 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:45.283 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:45.283 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 3264, 2023-07-14T11:29:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 195936 2023-07-14T11:29:45.284 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_messages": 337, 2023-07-14T11:29:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages": 328, 2023-07-14T11:29:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_bytes": 1318800, 2023-07-14T11:29:45.285 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_bytes": 187936, 2023-07-14T11:29:45.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_created_connections": 2, 2023-07-14T11:29:45.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_active_connections": 1, 2023-07-14T11:29:45.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_total_time": 0.048434615, 2023-07-14T11:29:45.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_send_time": 0.014582247, 2023-07-14T11:29:45.286 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_recv_time": 0.031331469, 2023-07-14T11:29:45.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_running_fast_dispatch_time": 0.008437805, 2023-07-14T11:29:45.287 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:45.287 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 327, 2023-07-14T11:29:45.288 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.009692037, 2023-07-14T11:29:45.288 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000029639 2023-07-14T11:29:45.288 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.289 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:45.289 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:45.289 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:45.289 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:45.289 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.290 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_recv_encrypted_bytes": 1318800, 2023-07-14T11:29:45.290 INFO:teuthology.orchestra.run.smithi045.stdout: "msgr_send_encrypted_bytes": 187936 2023-07-14T11:29:45.290 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.291 INFO:teuthology.orchestra.run.smithi045.stdout: "cct": { 2023-07-14T11:29:45.291 INFO:teuthology.orchestra.run.smithi045.stdout: "total_workers": 0, 2023-07-14T11:29:45.292 INFO:teuthology.orchestra.run.smithi045.stdout: "unhealthy_workers": 0 2023-07-14T11:29:45.292 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.292 INFO:teuthology.orchestra.run.smithi045.stdout: "mempool": { 2023-07-14T11:29:45.293 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:45.293 INFO:teuthology.orchestra.run.smithi045.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:45.294 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:45.294 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:45.294 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:45.295 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:45.295 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:45.295 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:45.296 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:45.296 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:45.297 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:45.297 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:45.297 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:45.298 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:45.298 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:45.299 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:45.299 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:45.299 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:45.300 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:45.300 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:45.301 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:45.301 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:45.301 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:45.302 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:45.302 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:45.303 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:45.303 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:45.303 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:45.304 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:45.304 INFO:teuthology.orchestra.run.smithi045.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:45.305 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:45.305 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_items": 0, 2023-07-14T11:29:45.305 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:45.306 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:45.306 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:45.307 INFO:teuthology.orchestra.run.smithi045.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:45.307 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_bytes": 148955, 2023-07-14T11:29:45.307 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_anon_items": 30, 2023-07-14T11:29:45.308 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:45.308 INFO:teuthology.orchestra.run.smithi045.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:45.309 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_bytes": 0, 2023-07-14T11:29:45.309 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_items": 0, 2023-07-14T11:29:45.309 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:45.310 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:45.310 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:45.311 INFO:teuthology.orchestra.run.smithi045.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:45.311 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_bytes": 0, 2023-07-14T11:29:45.311 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_items": 0, 2023-07-14T11:29:45.312 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:45.312 INFO:teuthology.orchestra.run.smithi045.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:45.313 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:45.313 INFO:teuthology.orchestra.run.smithi045.stdout: "pgmap_items": 0, 2023-07-14T11:29:45.313 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_bytes": 928, 2023-07-14T11:29:45.313 INFO:teuthology.orchestra.run.smithi045.stdout: "mds_co_items": 2, 2023-07-14T11:29:45.314 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:45.314 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_1_items": 0, 2023-07-14T11:29:45.314 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:45.314 INFO:teuthology.orchestra.run.smithi045.stdout: "unittest_2_items": 0 2023-07-14T11:29:45.314 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2023-07-14T11:29:45.315 INFO:teuthology.orchestra.run.smithi045.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:45.315 INFO:teuthology.orchestra.run.smithi045.stdout: "val": 0, 2023-07-14T11:29:45.315 INFO:teuthology.orchestra.run.smithi045.stdout: "max": 104857600, 2023-07-14T11:29:45.315 INFO:teuthology.orchestra.run.smithi045.stdout: "get_started": 0, 2023-07-14T11:29:45.316 INFO:teuthology.orchestra.run.smithi045.stdout: "get": 339, 2023-07-14T11:29:45.316 INFO:teuthology.orchestra.run.smithi045.stdout: "get_sum": 1271170, 2023-07-14T11:29:45.316 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:45.317 INFO:teuthology.orchestra.run.smithi045.stdout: "get_or_fail_success": 339, 2023-07-14T11:29:45.317 INFO:teuthology.orchestra.run.smithi045.stdout: "take": 0, 2023-07-14T11:29:45.317 INFO:teuthology.orchestra.run.smithi045.stdout: "take_sum": 0, 2023-07-14T11:29:45.317 INFO:teuthology.orchestra.run.smithi045.stdout: "put": 339, 2023-07-14T11:29:45.317 INFO:teuthology.orchestra.run.smithi045.stdout: "put_sum": 1271170, 2023-07-14T11:29:45.318 INFO:teuthology.orchestra.run.smithi045.stdout: "wait": { 2023-07-14T11:29:45.318 INFO:teuthology.orchestra.run.smithi045.stdout: "avgcount": 0, 2023-07-14T11:29:45.318 INFO:teuthology.orchestra.run.smithi045.stdout: "sum": 0.000000000, 2023-07-14T11:29:45.318 INFO:teuthology.orchestra.run.smithi045.stdout: "avgtime": 0.000000000 2023-07-14T11:29:45.318 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:45.319 INFO:teuthology.orchestra.run.smithi045.stdout: } 2023-07-14T11:29:45.319 INFO:teuthology.orchestra.run.smithi045.stdout:} 2023-07-14T11:29:45.816 WARNING:tasks.check_counter:Counter 'mds.exported' not found on daemon mds.k 2023-07-14T11:29:45.816 WARNING:tasks.check_counter:Counter 'mds.imported' not found on daemon mds.k 2023-07-14T11:29:45.817 WARNING:tasks.check_counter:Counter 'mds.root_rsnaps' not found on daemon mds.k 2023-07-14T11:29:45.817 WARNING:tasks.check_counter:Counter 'mds_server.req_mksnap_latency.avgcount' not found on daemon mds.k 2023-07-14T11:29:45.817 WARNING:tasks.check_counter:Counter 'mds_server.req_rmsnap_latency.avgcount' not found on daemon mds.k 2023-07-14T11:29:45.817 DEBUG:tasks.check_counter:Getting stats from c 2023-07-14T11:29:45.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.c perf dump 2023-07-14T11:29:46.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:45 smithi130 ceph-mon[129405]: pgmap v553: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:46.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:45 smithi040 ceph-mon[97954]: pgmap v553: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:46.064 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:45 smithi045 ceph-mon[128791]: pgmap v553: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:46.388 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:29:48.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:47 smithi130 ceph-mon[129405]: pgmap v554: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:48.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:47 smithi040 ceph-mon[97954]: pgmap v554: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:48.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:47 smithi045 ceph-mon[128791]: pgmap v554: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:49.266 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-07-14T11:29:49.267 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:49.267 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 2993, 2023-07-14T11:29:49.267 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 3684, 2023-07-14T11:29:49.268 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 3895511, 2023-07-14T11:29:49.268 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 4783395, 2023-07-14T11:29:49.268 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 18, 2023-07-14T11:29:49.269 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 6, 2023-07-14T11:29:49.269 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.181262928, 2023-07-14T11:29:49.269 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.067224074, 2023-07-14T11:29:49.269 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.123780670, 2023-07-14T11:29:49.269 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.021626771, 2023-07-14T11:29:49.270 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:49.270 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3682, 2023-07-14T11:29:49.270 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.190889945, 2023-07-14T11:29:49.270 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000051844 2023-07-14T11:29:49.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.271 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:49.271 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3054, 2023-07-14T11:29:49.271 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000569638, 2023-07-14T11:29:49.271 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000186 2023-07-14T11:29:49.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.272 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2755488, 2023-07-14T11:29:49.272 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 201104 2023-07-14T11:29:49.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.272 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:49.273 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 186, 2023-07-14T11:29:49.273 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 464, 2023-07-14T11:29:49.273 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 60937, 2023-07-14T11:29:49.273 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 906379, 2023-07-14T11:29:49.273 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 19, 2023-07-14T11:29:49.274 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 6, 2023-07-14T11:29:49.274 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.028518638, 2023-07-14T11:29:49.274 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.012323271, 2023-07-14T11:29:49.274 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.008937831, 2023-07-14T11:29:49.275 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.004229400, 2023-07-14T11:29:49.275 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:49.275 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 459, 2023-07-14T11:29:49.275 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.032079999, 2023-07-14T11:29:49.276 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000069891 2023-07-14T11:29:49.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.276 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:49.276 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 72, 2023-07-14T11:29:49.277 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000012494, 2023-07-14T11:29:49.277 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000173 2023-07-14T11:29:49.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.277 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 4272, 2023-07-14T11:29:49.278 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 595584 2023-07-14T11:29:49.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.279 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:49.279 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 302105, 2023-07-14T11:29:49.279 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 298135, 2023-07-14T11:29:49.280 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 53366744, 2023-07-14T11:29:49.280 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 50862011, 2023-07-14T11:29:49.280 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 9, 2023-07-14T11:29:49.281 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:49.281 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 8.001302033, 2023-07-14T11:29:49.281 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 3.139961929, 2023-07-14T11:29:49.281 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 5.514679302, 2023-07-14T11:29:49.281 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.007781733, 2023-07-14T11:29:49.282 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:49.282 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 298137, 2023-07-14T11:29:49.282 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 2.645805919, 2023-07-14T11:29:49.282 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000008874 2023-07-14T11:29:49.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.283 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:49.283 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 598019, 2023-07-14T11:29:49.283 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.049754558, 2023-07-14T11:29:49.283 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000083 2023-07-14T11:29:49.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.284 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:49.284 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:49.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.284 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2023-07-14T11:29:49.285 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2023-07-14T11:29:49.285 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2023-07-14T11:29:49.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.285 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1143, 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.423092256, 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000370159 2023-07-14T11:29:49.286 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.287 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:49.287 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:49.287 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:49.287 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 16, 2023-07-14T11:29:49.288 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000289501, 2023-07-14T11:29:49.288 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000018093 2023-07-14T11:29:49.288 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.288 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2023-07-14T11:29:49.289 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 461, 2023-07-14T11:29:49.289 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 429, 2023-07-14T11:29:49.289 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2023-07-14T11:29:49.289 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 429, 2023-07-14T11:29:49.290 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 367.632736324, 2023-07-14T11:29:49.290 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.856952765 2023-07-14T11:29:49.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.291 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2023-07-14T11:29:49.291 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 32, 2023-07-14T11:29:49.292 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 3, 2023-07-14T11:29:49.292 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:49.293 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 43, 2023-07-14T11:29:49.293 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 4, 2023-07-14T11:29:49.294 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2023-07-14T11:29:49.294 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 898, 2023-07-14T11:29:49.294 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 284, 2023-07-14T11:29:49.295 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 573, 2023-07-14T11:29:49.295 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 41, 2023-07-14T11:29:49.296 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 492, 2023-07-14T11:29:49.296 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 203, 2023-07-14T11:29:49.297 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 112, 2023-07-14T11:29:49.297 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 112, 2023-07-14T11:29:49.298 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 67, 2023-07-14T11:29:49.298 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 12535, 2023-07-14T11:29:49.299 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 9493, 2023-07-14T11:29:49.299 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 26, 2023-07-14T11:29:49.299 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 49, 2023-07-14T11:29:49.300 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 3, 2023-07-14T11:29:49.301 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:49.301 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 2840, 2023-07-14T11:29:49.302 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2023-07-14T11:29:49.302 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2023-07-14T11:29:49.302 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 3, 2023-07-14T11:29:49.302 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 168, 2023-07-14T11:29:49.302 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 19, 2023-07-14T11:29:49.303 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 650, 2023-07-14T11:29:49.303 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:49.303 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 246, 2023-07-14T11:29:49.303 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:49.303 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 1028, 2023-07-14T11:29:49.304 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 5852694522, 2023-07-14T11:29:49.304 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 4, 2023-07-14T11:29:49.304 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 246, 2023-07-14T11:29:49.304 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:49.305 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:49.305 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:49.305 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:49.305 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:49.305 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 246, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 34, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 466, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 787, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:49.306 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:49.307 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 1, 2023-07-14T11:29:49.307 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 826, 2023-07-14T11:29:49.307 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 147, 2023-07-14T11:29:49.307 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 32, 2023-07-14T11:29:49.307 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 165 2023-07-14T11:29:49.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.308 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2023-07-14T11:29:49.308 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2023-07-14T11:29:49.309 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:49.309 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2023-07-14T11:29:49.309 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 52, 2023-07-14T11:29:49.309 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 36, 2023-07-14T11:29:49.309 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 0, 2023-07-14T11:29:49.310 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:49.310 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:49.310 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 24, 2023-07-14T11:29:49.310 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 1, 2023-07-14T11:29:49.310 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 112, 2023-07-14T11:29:49.311 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 14, 2023-07-14T11:29:49.311 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:49.311 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:49.311 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:49.311 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2023-07-14T11:29:49.312 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2023-07-14T11:29:49.312 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:49.312 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 3, 2023-07-14T11:29:49.312 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2023-07-14T11:29:49.312 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 4, 2023-07-14T11:29:49.313 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:49.313 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2023-07-14T11:29:49.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.313 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2023-07-14T11:29:49.313 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 1608, 2023-07-14T11:29:49.314 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2023-07-14T11:29:49.314 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2023-07-14T11:29:49.314 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 1608, 2023-07-14T11:29:49.314 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2023-07-14T11:29:49.314 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2023-07-14T11:29:49.315 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 2, 2023-07-14T11:29:49.315 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2023-07-14T11:29:49.315 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2023-07-14T11:29:49.315 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 2, 2023-07-14T11:29:49.316 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2023-07-14T11:29:49.316 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2023-07-14T11:29:49.316 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2023-07-14T11:29:49.316 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 7150304, 2023-07-14T11:29:49.317 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 0, 2023-07-14T11:29:49.317 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2023-07-14T11:29:49.317 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 547, 2023-07-14T11:29:49.317 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 7.370483945, 2023-07-14T11:29:49.317 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.013474376 2023-07-14T11:29:49.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.318 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 0 2023-07-14T11:29:49.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.319 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2023-07-14T11:29:49.319 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 830, 2023-07-14T11:29:49.320 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 976, 2023-07-14T11:29:49.320 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 146, 2023-07-14T11:29:49.320 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 86, 2023-07-14T11:29:49.321 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 93, 2023-07-14T11:29:49.321 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 7, 2023-07-14T11:29:49.321 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 898, 2023-07-14T11:29:49.322 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 1118, 2023-07-14T11:29:49.322 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 220, 2023-07-14T11:29:49.322 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 112, 2023-07-14T11:29:49.322 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 783, 2023-07-14T11:29:49.322 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 671, 2023-07-14T11:29:49.323 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 62152, 2023-07-14T11:29:49.323 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 198964 2023-07-14T11:29:49.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.323 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2023-07-14T11:29:49.324 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 6768, 2023-07-14T11:29:49.324 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 671, 2023-07-14T11:29:49.324 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 461, 2023-07-14T11:29:49.324 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 64, 2023-07-14T11:29:49.324 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 3285, 2023-07-14T11:29:49.325 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2023-07-14T11:29:49.325 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 7, 2023-07-14T11:29:49.325 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.013479746, 2023-07-14T11:29:49.325 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.001925678 2023-07-14T11:29:49.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 19, 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.201175981, 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.010588209 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.326 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:49.327 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.327 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.327 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.327 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2023-07-14T11:29:49.328 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 27, 2023-07-14T11:29:49.328 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 4.942033230, 2023-07-14T11:29:49.328 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.183038267 2023-07-14T11:29:49.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 101, 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 1.321627604, 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.013085421 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.329 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:49.330 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.330 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.330 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.331 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2023-07-14T11:29:49.331 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.331 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.331 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.332 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2023-07-14T11:29:49.332 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.333 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.333 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.333 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:49.333 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.334 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.334 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.334 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:49.334 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.335 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.335 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.335 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2023-07-14T11:29:49.335 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.336 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.336 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.336 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2023-07-14T11:29:49.336 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.337 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.337 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.337 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2023-07-14T11:29:49.337 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.338 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.338 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.338 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2023-07-14T11:29:49.338 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.339 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.339 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.339 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2023-07-14T11:29:49.339 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 23, 2023-07-14T11:29:49.340 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 1.713322557, 2023-07-14T11:29:49.340 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.074492285 2023-07-14T11:29:49.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.340 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2023-07-14T11:29:49.340 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 102, 2023-07-14T11:29:49.341 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 5.302316367, 2023-07-14T11:29:49.341 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.051983493 2023-07-14T11:29:49.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.341 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2023-07-14T11:29:49.341 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 126, 2023-07-14T11:29:49.342 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 337.926353632, 2023-07-14T11:29:49.342 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 2.681955187 2023-07-14T11:29:49.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.342 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:49.342 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.343 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.343 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.343 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:49.343 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.344 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.344 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.344 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2023-07-14T11:29:49.344 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.345 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.345 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.345 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:49.346 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.346 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.346 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.346 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:49.347 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.347 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.347 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.348 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2023-07-14T11:29:49.348 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.348 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.348 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.349 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:49.372 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.372 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.373 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.373 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:49.374 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.374 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.374 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.375 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2023-07-14T11:29:49.375 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.375 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.375 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.376 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2023-07-14T11:29:49.376 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.376 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.376 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.377 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2023-07-14T11:29:49.377 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.377 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.378 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.378 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2023-07-14T11:29:49.379 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 24, 2023-07-14T11:29:49.379 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 16.212427207, 2023-07-14T11:29:49.380 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.675517800 2023-07-14T11:29:49.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.380 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:49.380 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:49.381 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:49.381 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.381 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.381 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.381 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.382 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2023-07-14T11:29:49.382 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 2, 2023-07-14T11:29:49.382 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 3, 2023-07-14T11:29:49.382 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 1, 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 2, 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 120, 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 60, 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 1030 2023-07-14T11:29:49.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.384 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2023-07-14T11:29:49.384 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:49.384 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:49.384 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2023-07-14T11:29:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 647206, 2023-07-14T11:29:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 812, 2023-07-14T11:29:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2023-07-14T11:29:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2023-07-14T11:29:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 183, 2023-07-14T11:29:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2023-07-14T11:29:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 2791458, 2023-07-14T11:29:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 34890, 2023-07-14T11:29:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2023-07-14T11:29:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2023-07-14T11:29:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2023-07-14T11:29:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2023-07-14T11:29:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2023-07-14T11:29:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 942, 2023-07-14T11:29:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 3305922, 2023-07-14T11:29:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 33, 2023-07-14T11:29:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 942, 2023-07-14T11:29:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2023-07-14T11:29:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 942, 2023-07-14T11:29:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 7.803850325, 2023-07-14T11:29:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.008284342 2023-07-14T11:29:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2023-07-14T11:29:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2023-07-14T11:29:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 942, 2023-07-14T11:29:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 1062 2023-07-14T11:29:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 942, 2023-07-14T11:29:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 250, 2023-07-14T11:29:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 692, 2023-07-14T11:29:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2023-07-14T11:29:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2023-07-14T11:29:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 0, 2023-07-14T11:29:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 10, 2023-07-14T11:29:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1, 2023-07-14T11:29:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 548, 2023-07-14T11:29:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 27, 2023-07-14T11:29:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2023-07-14T11:29:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2023-07-14T11:29:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2023-07-14T11:29:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2023-07-14T11:29:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 18, 2023-07-14T11:29:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2023-07-14T11:29:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 246, 2023-07-14T11:29:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 20, 2023-07-14T11:29:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2023-07-14T11:29:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2023-07-14T11:29:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2023-07-14T11:29:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2023-07-14T11:29:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2023-07-14T11:29:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2023-07-14T11:29:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2023-07-14T11:29:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2023-07-14T11:29:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2023-07-14T11:29:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2023-07-14T11:29:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2023-07-14T11:29:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2023-07-14T11:29:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2023-07-14T11:29:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2023-07-14T11:29:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2023-07-14T11:29:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2023-07-14T11:29:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2023-07-14T11:29:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2023-07-14T11:29:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2023-07-14T11:29:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2023-07-14T11:29:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2023-07-14T11:29:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 115, 2023-07-14T11:29:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2023-07-14T11:29:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 25, 2023-07-14T11:29:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 12, 2023-07-14T11:29:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 12, 2023-07-14T11:29:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 0, 2023-07-14T11:29:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2023-07-14T11:29:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 155, 2023-07-14T11:29:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 6, 2023-07-14T11:29:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 31 2023-07-14T11:29:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2023-07-14T11:29:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 1, 2023-07-14T11:29:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2023-07-14T11:29:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2023-07-14T11:29:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2023-07-14T11:29:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 577 2023-07-14T11:29:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2023-07-14T11:29:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 2, 2023-07-14T11:29:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2023-07-14T11:29:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 1, 2023-07-14T11:29:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 1, 2023-07-14T11:29:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 305280, 2023-07-14T11:29:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 29812786, 2023-07-14T11:29:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 305280, 2023-07-14T11:29:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 305280, 2023-07-14T11:29:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 29812786, 2023-07-14T11:29:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 942, 2023-07-14T11:29:49.424 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 3306015, 2023-07-14T11:29:49.424 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 942, 2023-07-14T11:29:49.424 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 3306015, 2023-07-14T11:29:49.424 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2023-07-14T11:29:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 942, 2023-07-14T11:29:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 942, 2023-07-14T11:29:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 942, 2023-07-14T11:29:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 942, 2023-07-14T11:29:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 1, 2023-07-14T11:29:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 141, 2023-07-14T11:29:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 1, 2023-07-14T11:29:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 1, 2023-07-14T11:29:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 141, 2023-07-14T11:29:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x558d6634c0a0": { 2023-07-14T11:29:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 1608, 2023-07-14T11:29:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 2956000, 2023-07-14T11:29:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 1608, 2023-07-14T11:29:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 547, 2023-07-14T11:29:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 2956000, 2023-07-14T11:29:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:49.437 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-07-14T11:29:49.851 INFO:tasks.check_counter:Daemon mds.c mds.exported=3 2023-07-14T11:29:49.851 INFO:tasks.check_counter:Daemon mds.c mds.imported=19 2023-07-14T11:29:49.852 INFO:tasks.check_counter:Daemon mds.c mds.root_rsnaps=4 2023-07-14T11:29:49.852 INFO:tasks.check_counter:Daemon mds.c mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:49.852 INFO:tasks.check_counter:Daemon mds.c mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:49.852 DEBUG:tasks.check_counter:Getting stats from f 2023-07-14T11:29:49.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.f perf dump 2023-07-14T11:29:50.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:49 smithi130 ceph-mon[129405]: pgmap v555: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:50.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:49 smithi040 ceph-mon[97954]: pgmap v555: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:50.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:49 smithi045 ceph-mon[128791]: pgmap v555: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:50.422 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:29:51.784 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-07-14T11:29:51.785 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:51.785 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 5162837, 2023-07-14T11:29:51.785 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 5163118, 2023-07-14T11:29:51.786 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 883966560, 2023-07-14T11:29:51.786 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 820226552, 2023-07-14T11:29:51.786 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 20, 2023-07-14T11:29:51.786 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 8, 2023-07-14T11:29:51.786 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 126.827837600, 2023-07-14T11:29:51.787 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 50.796655529, 2023-07-14T11:29:51.787 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 84.366018212, 2023-07-14T11:29:51.787 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.014744098, 2023-07-14T11:29:51.788 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:51.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 5163116, 2023-07-14T11:29:51.788 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 41.682846318, 2023-07-14T11:29:51.788 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000008073 2023-07-14T11:29:51.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.789 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:51.789 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 9894482, 2023-07-14T11:29:51.789 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.920697588, 2023-07-14T11:29:51.790 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000093 2023-07-14T11:29:51.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.790 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 3104, 2023-07-14T11:29:51.791 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 462912 2023-07-14T11:29:51.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.791 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:51.791 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 2261, 2023-07-14T11:29:51.791 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 3294, 2023-07-14T11:29:51.792 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 1338189, 2023-07-14T11:29:51.792 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 2538608, 2023-07-14T11:29:51.792 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 15, 2023-07-14T11:29:51.792 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 6, 2023-07-14T11:29:51.792 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.102912907, 2023-07-14T11:29:51.793 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.044710678, 2023-07-14T11:29:51.793 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.109806746, 2023-07-14T11:29:51.793 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.002275069, 2023-07-14T11:29:51.793 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:51.793 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3292, 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.100825307, 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000030627 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3812, 2023-07-14T11:29:51.794 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000674839, 2023-07-14T11:29:51.795 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000177 2023-07-14T11:29:51.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.795 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 160, 2023-07-14T11:29:51.795 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 123120 2023-07-14T11:29:51.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.796 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:51.796 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 1258, 2023-07-14T11:29:51.796 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 1019, 2023-07-14T11:29:51.796 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 3267889, 2023-07-14T11:29:51.797 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 988929, 2023-07-14T11:29:51.797 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 10, 2023-07-14T11:29:51.797 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:51.797 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.092274786, 2023-07-14T11:29:51.797 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.031836396, 2023-07-14T11:29:51.798 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.066713365, 2023-07-14T11:29:51.798 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.010061973, 2023-07-14T11:29:51.798 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:51.798 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1018, 2023-07-14T11:29:51.799 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.050520980, 2023-07-14T11:29:51.799 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000049627 2023-07-14T11:29:51.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.799 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:51.799 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 902, 2023-07-14T11:29:51.800 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000147080, 2023-07-14T11:29:51.800 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000163 2023-07-14T11:29:51.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.800 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2740736, 2023-07-14T11:29:51.800 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 211584 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.801 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2023-07-14T11:29:51.802 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:51.802 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:51.802 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 768, 2023-07-14T11:29:51.802 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.421029044, 2023-07-14T11:29:51.802 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000548214 2023-07-14T11:29:51.803 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.803 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:51.803 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:51.803 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:51.804 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 18, 2023-07-14T11:29:51.804 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000317773, 2023-07-14T11:29:51.804 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000017654 2023-07-14T11:29:51.804 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.805 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2023-07-14T11:29:51.805 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 205, 2023-07-14T11:29:51.805 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 189, 2023-07-14T11:29:51.805 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2023-07-14T11:29:51.806 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 189, 2023-07-14T11:29:51.806 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 102.644173595, 2023-07-14T11:29:51.806 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.543090865 2023-07-14T11:29:51.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.806 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 16, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 5, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 51, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 5, 2023-07-14T11:29:51.807 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2023-07-14T11:29:51.808 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 698, 2023-07-14T11:29:51.808 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 228, 2023-07-14T11:29:51.808 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 457, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 13, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 370, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 257, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 104, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 104, 2023-07-14T11:29:51.809 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 48, 2023-07-14T11:29:51.810 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 1118, 2023-07-14T11:29:51.810 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 949, 2023-07-14T11:29:51.810 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 9, 2023-07-14T11:29:51.810 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 16, 2023-07-14T11:29:51.810 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:51.811 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:51.811 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 127, 2023-07-14T11:29:51.811 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2023-07-14T11:29:51.811 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2023-07-14T11:29:51.812 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 10, 2023-07-14T11:29:51.812 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 240, 2023-07-14T11:29:51.812 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 9, 2023-07-14T11:29:51.813 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 525, 2023-07-14T11:29:51.813 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:51.814 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 183, 2023-07-14T11:29:51.814 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 3, 2023-07-14T11:29:51.815 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 1028, 2023-07-14T11:29:51.815 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 5852694522, 2023-07-14T11:29:51.816 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 4, 2023-07-14T11:29:51.816 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 183, 2023-07-14T11:29:51.817 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:51.817 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:51.817 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:51.818 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:51.819 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:51.819 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:51.820 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 183, 2023-07-14T11:29:51.820 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 1, 2023-07-14T11:29:51.820 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 675, 2023-07-14T11:29:51.821 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 985, 2023-07-14T11:29:51.821 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:51.821 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:51.822 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:51.822 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 995, 2023-07-14T11:29:51.823 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 14, 2023-07-14T11:29:51.823 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 16, 2023-07-14T11:29:51.823 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 128 2023-07-14T11:29:51.823 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.824 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2023-07-14T11:29:51.824 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2023-07-14T11:29:51.824 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:51.824 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2023-07-14T11:29:51.824 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 23, 2023-07-14T11:29:51.825 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 6, 2023-07-14T11:29:51.825 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 1, 2023-07-14T11:29:51.825 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:51.825 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:51.825 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 4, 2023-07-14T11:29:51.826 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 1, 2023-07-14T11:29:51.826 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 27, 2023-07-14T11:29:51.826 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 3, 2023-07-14T11:29:51.826 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:51.826 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:51.827 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:51.827 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2023-07-14T11:29:51.827 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2023-07-14T11:29:51.827 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:51.828 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 24, 2023-07-14T11:29:51.828 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2023-07-14T11:29:51.828 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 5, 2023-07-14T11:29:51.828 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:51.829 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2023-07-14T11:29:51.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.829 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2023-07-14T11:29:51.829 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 1542, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 1542, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2023-07-14T11:29:51.830 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 2, 2023-07-14T11:29:51.831 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2023-07-14T11:29:51.831 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2023-07-14T11:29:51.831 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 2, 2023-07-14T11:29:51.831 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2023-07-14T11:29:51.832 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2023-07-14T11:29:51.832 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2023-07-14T11:29:51.832 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 6785619, 2023-07-14T11:29:51.832 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 0, 2023-07-14T11:29:51.832 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2023-07-14T11:29:51.833 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 404, 2023-07-14T11:29:51.833 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 11.869579218, 2023-07-14T11:29:51.833 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.029380146 2023-07-14T11:29:51.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.833 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 0 2023-07-14T11:29:51.834 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.834 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2023-07-14T11:29:51.834 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 694, 2023-07-14T11:29:51.834 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 922, 2023-07-14T11:29:51.834 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 228, 2023-07-14T11:29:51.835 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 66, 2023-07-14T11:29:51.835 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 72, 2023-07-14T11:29:51.835 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 6, 2023-07-14T11:29:51.835 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 698, 2023-07-14T11:29:51.835 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 967, 2023-07-14T11:29:51.836 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 269, 2023-07-14T11:29:51.836 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 104, 2023-07-14T11:29:51.836 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 881, 2023-07-14T11:29:51.837 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 777, 2023-07-14T11:29:51.837 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 63428, 2023-07-14T11:29:51.837 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 198964 2023-07-14T11:29:51.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.837 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2023-07-14T11:29:51.838 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 719, 2023-07-14T11:29:51.838 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 150, 2023-07-14T11:29:51.838 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 205, 2023-07-14T11:29:51.838 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 42, 2023-07-14T11:29:51.838 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 326, 2023-07-14T11:29:51.839 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2023-07-14T11:29:51.839 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.839 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.839 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.839 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.840 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2023-07-14T11:29:51.840 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3, 2023-07-14T11:29:51.840 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000653649, 2023-07-14T11:29:51.840 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000217883 2023-07-14T11:29:51.841 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.841 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:51.841 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.841 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.842 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.842 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.842 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2023-07-14T11:29:51.842 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 4, 2023-07-14T11:29:51.842 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.030285513, 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.007571378 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 23, 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.006605008, 2023-07-14T11:29:51.843 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000287174 2023-07-14T11:29:51.844 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.844 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:51.844 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.844 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.845 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.846 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.846 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2023-07-14T11:29:51.846 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.846 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.846 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.847 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.847 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:51.847 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.847 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.847 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.848 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.848 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:51.848 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.849 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.849 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.849 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.849 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2023-07-14T11:29:51.850 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.850 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.850 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.851 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.851 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2023-07-14T11:29:51.852 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.852 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.852 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.853 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.853 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2023-07-14T11:29:51.853 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.853 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.853 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.854 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.854 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2023-07-14T11:29:51.854 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.854 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.855 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.855 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.855 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2023-07-14T11:29:51.855 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1, 2023-07-14T11:29:51.855 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000423212, 2023-07-14T11:29:51.856 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000423212 2023-07-14T11:29:51.856 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.856 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2023-07-14T11:29:51.856 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 122, 2023-07-14T11:29:51.856 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 1.499472070, 2023-07-14T11:29:51.857 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.012290754 2023-07-14T11:29:51.857 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.857 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2023-07-14T11:29:51.857 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 32, 2023-07-14T11:29:51.857 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 101.093093261, 2023-07-14T11:29:51.858 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 3.159159164 2023-07-14T11:29:51.858 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.858 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:51.859 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.859 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.859 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.859 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.859 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:51.860 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.860 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.860 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.860 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.861 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2023-07-14T11:29:51.861 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.861 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.862 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.862 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:51.862 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.862 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.863 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.863 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:51.863 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.863 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.864 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.864 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2023-07-14T11:29:51.864 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.865 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.865 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.865 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.865 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:51.866 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.866 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.866 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.867 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.867 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:51.867 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.868 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.868 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.869 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2023-07-14T11:29:51.869 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.869 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.869 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.870 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2023-07-14T11:29:51.870 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.870 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.870 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.870 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.871 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2023-07-14T11:29:51.871 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.871 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.871 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.872 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.872 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2023-07-14T11:29:51.872 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 4, 2023-07-14T11:29:51.872 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.013640882, 2023-07-14T11:29:51.873 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.003410220 2023-07-14T11:29:51.873 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.873 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:51.873 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:51.873 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:51.874 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.874 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.874 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.874 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.874 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.875 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2023-07-14T11:29:51.875 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 2, 2023-07-14T11:29:51.875 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 2, 2023-07-14T11:29:51.876 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 0, 2023-07-14T11:29:51.876 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 2, 2023-07-14T11:29:51.876 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2023-07-14T11:29:51.876 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 104, 2023-07-14T11:29:51.876 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 52, 2023-07-14T11:29:51.877 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 2236 2023-07-14T11:29:51.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.877 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2023-07-14T11:29:51.877 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:51.877 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:51.878 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:51.878 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:51.878 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:51.878 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:51.878 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:51.879 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:51.879 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:51.879 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:51.879 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:51.879 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:51.880 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:51.880 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:51.880 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:51.880 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:51.880 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:51.881 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:51.881 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:51.881 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:51.881 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:51.881 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:51.882 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:51.882 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:51.882 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:51.883 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:51.883 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:51.883 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:51.883 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:51.884 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:51.884 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:51.885 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2023-07-14T11:29:51.885 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:51.885 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:51.886 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:51.886 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:51.887 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 415961, 2023-07-14T11:29:51.887 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 565, 2023-07-14T11:29:51.888 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:51.888 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:51.888 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2023-07-14T11:29:51.889 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2023-07-14T11:29:51.889 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:51.890 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:51.890 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:51.890 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:51.891 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:51.891 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 183, 2023-07-14T11:29:51.891 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:51.891 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:51.891 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:51.892 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2023-07-14T11:29:51.892 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 2224603, 2023-07-14T11:29:51.892 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 22445, 2023-07-14T11:29:51.893 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:51.893 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2023-07-14T11:29:51.893 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:51.893 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2023-07-14T11:29:51.894 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.894 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2023-07-14T11:29:51.894 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2023-07-14T11:29:51.895 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2023-07-14T11:29:51.895 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 729, 2023-07-14T11:29:51.895 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 3013085, 2023-07-14T11:29:51.895 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 33, 2023-07-14T11:29:51.896 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 729, 2023-07-14T11:29:51.896 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2023-07-14T11:29:51.896 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 729, 2023-07-14T11:29:51.897 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 12.203765371, 2023-07-14T11:29:51.897 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.016740418 2023-07-14T11:29:51.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.898 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2023-07-14T11:29:51.898 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2023-07-14T11:29:51.898 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 729, 2023-07-14T11:29:51.898 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 835 2023-07-14T11:29:51.898 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.899 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 729, 2023-07-14T11:29:51.899 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 189, 2023-07-14T11:29:51.899 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 540, 2023-07-14T11:29:51.899 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2023-07-14T11:29:51.900 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2023-07-14T11:29:51.900 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 0, 2023-07-14T11:29:51.900 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 10, 2023-07-14T11:29:51.900 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1, 2023-07-14T11:29:51.901 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 405, 2023-07-14T11:29:51.901 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 23, 2023-07-14T11:29:51.901 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2023-07-14T11:29:51.902 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2023-07-14T11:29:51.902 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 1, 2023-07-14T11:29:51.903 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2023-07-14T11:29:51.903 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 20, 2023-07-14T11:29:51.903 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2023-07-14T11:29:51.904 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:51.904 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:51.904 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 183, 2023-07-14T11:29:51.905 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 20, 2023-07-14T11:29:51.905 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:51.905 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:51.906 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:51.906 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2023-07-14T11:29:51.906 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2023-07-14T11:29:51.907 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2023-07-14T11:29:51.907 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:51.908 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2023-07-14T11:29:51.908 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:51.908 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2023-07-14T11:29:51.909 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2023-07-14T11:29:51.909 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2023-07-14T11:29:51.909 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2023-07-14T11:29:51.910 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2023-07-14T11:29:51.910 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2023-07-14T11:29:51.910 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2023-07-14T11:29:51.911 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2023-07-14T11:29:51.911 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2023-07-14T11:29:51.911 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2023-07-14T11:29:51.911 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2023-07-14T11:29:51.911 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2023-07-14T11:29:51.912 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2023-07-14T11:29:51.912 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2023-07-14T11:29:51.912 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2023-07-14T11:29:51.912 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2023-07-14T11:29:51.912 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2023-07-14T11:29:51.913 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 115, 2023-07-14T11:29:51.913 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2023-07-14T11:29:51.913 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 25, 2023-07-14T11:29:51.913 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 12, 2023-07-14T11:29:51.913 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 12, 2023-07-14T11:29:51.914 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 0, 2023-07-14T11:29:51.914 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2023-07-14T11:29:51.914 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 145, 2023-07-14T11:29:51.914 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 10, 2023-07-14T11:29:51.915 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 17 2023-07-14T11:29:51.915 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.915 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2023-07-14T11:29:51.915 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 1, 2023-07-14T11:29:51.915 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:51.916 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2023-07-14T11:29:51.916 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2023-07-14T11:29:51.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.916 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2023-07-14T11:29:51.916 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 256 2023-07-14T11:29:51.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.917 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2023-07-14T11:29:51.917 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:51.917 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 4, 2023-07-14T11:29:51.917 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2023-07-14T11:29:51.918 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 1, 2023-07-14T11:29:51.918 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 1, 2023-07-14T11:29:51.918 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:51.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.918 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:51.919 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:51.919 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:51.919 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:51.919 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 5166353, 2023-07-14T11:29:51.919 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 423566185, 2023-07-14T11:29:51.920 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:51.920 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 5166353, 2023-07-14T11:29:51.920 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:51.920 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:51.920 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 5166353, 2023-07-14T11:29:51.921 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 423566185, 2023-07-14T11:29:51.921 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:51.921 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.921 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.921 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.922 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.922 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.922 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:51.922 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:51.922 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:51.923 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 729, 2023-07-14T11:29:51.924 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 3013114, 2023-07-14T11:29:51.924 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 729, 2023-07-14T11:29:51.924 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 3013114, 2023-07-14T11:29:51.925 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:51.925 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.925 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.925 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.925 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.926 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.926 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:51.926 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:51.926 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2023-07-14T11:29:51.927 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:51.927 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:51.927 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:51.928 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:51.928 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:51.928 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 729, 2023-07-14T11:29:51.929 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 729, 2023-07-14T11:29:51.929 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 729, 2023-07-14T11:29:51.930 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 729, 2023-07-14T11:29:51.930 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:51.930 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.930 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.931 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.931 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.932 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:51.932 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 1, 2023-07-14T11:29:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 109, 2023-07-14T11:29:51.933 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:51.934 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 1, 2023-07-14T11:29:51.934 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:51.934 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:51.934 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 1, 2023-07-14T11:29:51.934 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 109, 2023-07-14T11:29:51.935 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:51.935 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.935 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.935 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.936 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:51.936 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x56503e1700a0": { 2023-07-14T11:29:51.936 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:51.936 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:51.937 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:51.937 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 1542, 2023-07-14T11:29:51.937 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 2591315, 2023-07-14T11:29:51.938 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:51.938 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 1542, 2023-07-14T11:29:51.938 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:51.939 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:51.939 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 404, 2023-07-14T11:29:51.939 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 2591315, 2023-07-14T11:29:51.939 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:51.939 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:51.940 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:51.940 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:51.940 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.940 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:51.940 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-07-14T11:29:52.199 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:51 smithi130 ceph-mon[129405]: pgmap v556: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:52.289 INFO:tasks.check_counter:Daemon mds.f mds.exported=10 2023-07-14T11:29:52.290 INFO:tasks.check_counter:Daemon mds.f mds.imported=9 2023-07-14T11:29:52.292 INFO:tasks.check_counter:Daemon mds.f mds.root_rsnaps=4 2023-07-14T11:29:52.292 INFO:tasks.check_counter:Daemon mds.f mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:52.293 INFO:tasks.check_counter:Daemon mds.f mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:52.293 DEBUG:tasks.check_counter:Getting stats from i 2023-07-14T11:29:52.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.i perf dump 2023-07-14T11:29:52.300 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:51 smithi040 ceph-mon[97954]: pgmap v556: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:52.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:51 smithi045 ceph-mon[128791]: pgmap v556: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:52.878 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:29:54.226 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-07-14T11:29:54.227 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:54.227 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 1776, 2023-07-14T11:29:54.227 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 1535, 2023-07-14T11:29:54.228 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 3145920, 2023-07-14T11:29:54.228 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 558653, 2023-07-14T11:29:54.228 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:54.229 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 1, 2023-07-14T11:29:54.229 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.146025482, 2023-07-14T11:29:54.230 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.039453549, 2023-07-14T11:29:54.230 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.067066374, 2023-07-14T11:29:54.230 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.042669699, 2023-07-14T11:29:54.231 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:54.231 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1534, 2023-07-14T11:29:54.232 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.045111695, 2023-07-14T11:29:54.232 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000029407 2023-07-14T11:29:54.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.233 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:54.233 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.234 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.234 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.235 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2749776, 2023-07-14T11:29:54.235 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 191984 2023-07-14T11:29:54.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.236 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:54.236 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 5, 2023-07-14T11:29:54.237 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 251, 2023-07-14T11:29:54.237 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 1036, 2023-07-14T11:29:54.237 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 661308, 2023-07-14T11:29:54.238 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 8, 2023-07-14T11:29:54.238 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 4, 2023-07-14T11:29:54.238 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.011404615, 2023-07-14T11:29:54.239 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.006908341, 2023-07-14T11:29:54.239 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.002310096, 2023-07-14T11:29:54.239 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.000072728, 2023-07-14T11:29:54.240 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:54.240 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 248, 2023-07-14T11:29:54.241 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.019519216, 2023-07-14T11:29:54.241 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000078706 2023-07-14T11:29:54.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.242 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:54.242 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.243 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.243 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.244 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 480, 2023-07-14T11:29:54.244 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 660624 2023-07-14T11:29:54.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.245 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:54.245 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 2381, 2023-07-14T11:29:54.246 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 2381, 2023-07-14T11:29:54.246 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 3995548, 2023-07-14T11:29:54.246 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 739702, 2023-07-14T11:29:54.247 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:54.247 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:54.248 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.133666966, 2023-07-14T11:29:54.248 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.038138506, 2023-07-14T11:29:54.248 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.033351355, 2023-07-14T11:29:54.249 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.053825585, 2023-07-14T11:29:54.249 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:54.249 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2380, 2023-07-14T11:29:54.250 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.053060501, 2023-07-14T11:29:54.251 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000022294 2023-07-14T11:29:54.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.251 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:54.252 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.252 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.252 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.253 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2944, 2023-07-14T11:29:54.254 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 448 2023-07-14T11:29:54.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.254 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2023-07-14T11:29:54.255 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2023-07-14T11:29:54.255 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2023-07-14T11:29:54.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.256 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2023-07-14T11:29:54.256 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:54.257 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:54.257 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 6043, 2023-07-14T11:29:54.257 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.133034209, 2023-07-14T11:29:54.258 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000022014 2023-07-14T11:29:54.258 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.259 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:54.259 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:54.259 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:54.260 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.260 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.261 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.261 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.262 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2023-07-14T11:29:54.262 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 0, 2023-07-14T11:29:54.263 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 0, 2023-07-14T11:29:54.263 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2023-07-14T11:29:54.263 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.264 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.264 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.265 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2023-07-14T11:29:54.265 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 0, 2023-07-14T11:29:54.266 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:54.266 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:54.267 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 0, 2023-07-14T11:29:54.267 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 0, 2023-07-14T11:29:54.267 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2023-07-14T11:29:54.268 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 475, 2023-07-14T11:29:54.268 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 62, 2023-07-14T11:29:54.268 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 161, 2023-07-14T11:29:54.269 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 252, 2023-07-14T11:29:54.269 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 385, 2023-07-14T11:29:54.270 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 0, 2023-07-14T11:29:54.270 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 0, 2023-07-14T11:29:54.270 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 0, 2023-07-14T11:29:54.271 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 11, 2023-07-14T11:29:54.271 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 0, 2023-07-14T11:29:54.271 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 0, 2023-07-14T11:29:54.272 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 0, 2023-07-14T11:29:54.272 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 0, 2023-07-14T11:29:54.273 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:54.273 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:54.273 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 0, 2023-07-14T11:29:54.274 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2023-07-14T11:29:54.274 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2023-07-14T11:29:54.274 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 0, 2023-07-14T11:29:54.275 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 0, 2023-07-14T11:29:54.275 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 0, 2023-07-14T11:29:54.276 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 0, 2023-07-14T11:29:54.276 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:54.276 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 0, 2023-07-14T11:29:54.277 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:54.277 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 0, 2023-07-14T11:29:54.277 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 0, 2023-07-14T11:29:54.278 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 0, 2023-07-14T11:29:54.278 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 0, 2023-07-14T11:29:54.279 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:54.279 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:54.279 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:54.280 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:54.280 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:54.280 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:54.281 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 0, 2023-07-14T11:29:54.281 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 0, 2023-07-14T11:29:54.282 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 0, 2023-07-14T11:29:54.282 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 0, 2023-07-14T11:29:54.282 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:54.282 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:54.283 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:54.283 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 0, 2023-07-14T11:29:54.283 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 0, 2023-07-14T11:29:54.283 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 0, 2023-07-14T11:29:54.283 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 0 2023-07-14T11:29:54.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.284 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2023-07-14T11:29:54.284 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2023-07-14T11:29:54.284 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:54.284 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2023-07-14T11:29:54.285 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 0, 2023-07-14T11:29:54.285 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 0, 2023-07-14T11:29:54.285 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 0, 2023-07-14T11:29:54.285 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:54.285 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:54.286 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 0, 2023-07-14T11:29:54.286 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 0, 2023-07-14T11:29:54.286 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 0, 2023-07-14T11:29:54.286 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 0, 2023-07-14T11:29:54.286 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:54.287 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:54.287 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:54.287 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2023-07-14T11:29:54.287 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2023-07-14T11:29:54.287 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 0, 2023-07-14T11:29:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2023-07-14T11:29:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 0, 2023-07-14T11:29:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2023-07-14T11:29:54.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.289 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2023-07-14T11:29:54.289 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 0, 2023-07-14T11:29:54.289 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2023-07-14T11:29:54.289 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2023-07-14T11:29:54.290 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 0, 2023-07-14T11:29:54.290 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2023-07-14T11:29:54.290 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2023-07-14T11:29:54.290 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 0, 2023-07-14T11:29:54.290 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 2, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2023-07-14T11:29:54.291 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 0, 2023-07-14T11:29:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 7582083, 2023-07-14T11:29:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2023-07-14T11:29:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.293 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 1549 2023-07-14T11:29:54.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.293 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2023-07-14T11:29:54.293 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 270, 2023-07-14T11:29:54.294 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 317, 2023-07-14T11:29:54.294 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 47, 2023-07-14T11:29:54.294 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 17, 2023-07-14T11:29:54.294 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 23, 2023-07-14T11:29:54.294 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 6, 2023-07-14T11:29:54.295 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 475, 2023-07-14T11:29:54.295 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 608, 2023-07-14T11:29:54.295 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 133, 2023-07-14T11:29:54.295 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 0, 2023-07-14T11:29:54.295 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 0, 2023-07-14T11:29:54.296 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 0, 2023-07-14T11:29:54.296 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 51640, 2023-07-14T11:29:54.296 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 207156 2023-07-14T11:29:54.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.296 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 0, 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 0, 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 0, 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 0, 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 0, 2023-07-14T11:29:54.297 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2023-07-14T11:29:54.298 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.298 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.298 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.298 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2023-07-14T11:29:54.299 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.299 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.299 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.299 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:54.300 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.300 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.300 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.300 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2023-07-14T11:29:54.301 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.302 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.302 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.302 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:54.302 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.303 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.303 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.303 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2023-07-14T11:29:54.303 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.304 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.304 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.304 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2023-07-14T11:29:54.304 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.305 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.305 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.305 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:54.305 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.306 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.307 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.307 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2023-07-14T11:29:54.307 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.307 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.308 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.308 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2023-07-14T11:29:54.308 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.308 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.309 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.309 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2023-07-14T11:29:54.309 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.309 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.310 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.310 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2023-07-14T11:29:54.310 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.310 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.311 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.311 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2023-07-14T11:29:54.311 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.311 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.312 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.313 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2023-07-14T11:29:54.313 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.313 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.313 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.314 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:54.314 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.314 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.314 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.315 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:54.315 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.315 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.315 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.316 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2023-07-14T11:29:54.316 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.316 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.316 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.318 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:54.318 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.318 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.318 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.319 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2023-07-14T11:29:54.319 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.319 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.319 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.320 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:54.320 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.320 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.320 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.321 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:54.321 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.321 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.321 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.322 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2023-07-14T11:29:54.323 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.323 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.323 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.323 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2023-07-14T11:29:54.324 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.324 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.324 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.324 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2023-07-14T11:29:54.325 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.325 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.325 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.325 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.326 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.327 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2023-07-14T11:29:54.327 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 2, 2023-07-14T11:29:54.327 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 4, 2023-07-14T11:29:54.327 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 2, 2023-07-14T11:29:54.328 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 4, 2023-07-14T11:29:54.328 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2023-07-14T11:29:54.328 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 0, 2023-07-14T11:29:54.328 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 0, 2023-07-14T11:29:54.328 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 5873 2023-07-14T11:29:54.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.329 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2023-07-14T11:29:54.329 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:54.329 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:54.329 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:54.330 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:54.331 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:54.331 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:54.331 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:54.331 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:54.331 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:54.332 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:54.332 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:54.332 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:54.332 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:54.332 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:54.333 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:54.333 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:54.333 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:54.333 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:54.333 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:54.334 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:54.334 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:54.334 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:54.334 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:54.334 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:54.335 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:54.336 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:54.336 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 196641, 2023-07-14T11:29:54.336 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 442, 2023-07-14T11:29:54.336 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:54.336 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:54.337 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2023-07-14T11:29:54.337 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2023-07-14T11:29:54.337 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:54.337 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:54.337 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:54.338 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:54.338 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:54.338 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 183, 2023-07-14T11:29:54.338 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:54.338 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:54.339 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:54.339 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2023-07-14T11:29:54.339 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 1103246, 2023-07-14T11:29:54.339 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 20978, 2023-07-14T11:29:54.339 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2023-07-14T11:29:54.340 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2023-07-14T11:29:54.341 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2023-07-14T11:29:54.341 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 3558, 2023-07-14T11:29:54.341 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 21, 2023-07-14T11:29:54.341 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 3, 2023-07-14T11:29:54.341 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 3558, 2023-07-14T11:29:54.342 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2023-07-14T11:29:54.342 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3558, 2023-07-14T11:29:54.342 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 1.682969535, 2023-07-14T11:29:54.342 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000473009 2023-07-14T11:29:54.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.343 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2023-07-14T11:29:54.343 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2023-07-14T11:29:54.343 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3558, 2023-07-14T11:29:54.343 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 3559 2023-07-14T11:29:54.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.344 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 3558, 2023-07-14T11:29:54.344 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 3558, 2023-07-14T11:29:54.344 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 0, 2023-07-14T11:29:54.344 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2023-07-14T11:29:54.344 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 2243, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 0, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1313, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 0, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 0, 2023-07-14T11:29:54.345 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2023-07-14T11:29:54.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2023-07-14T11:29:54.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2023-07-14T11:29:54.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2023-07-14T11:29:54.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 0, 2023-07-14T11:29:54.346 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2023-07-14T11:29:54.347 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:54.347 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:54.347 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 1, 2023-07-14T11:29:54.347 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 0, 2023-07-14T11:29:54.347 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:54.348 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:54.348 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:54.348 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2023-07-14T11:29:54.348 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2023-07-14T11:29:54.348 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2023-07-14T11:29:54.349 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2023-07-14T11:29:54.350 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2023-07-14T11:29:54.350 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2023-07-14T11:29:54.350 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2023-07-14T11:29:54.350 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2023-07-14T11:29:54.350 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2023-07-14T11:29:54.351 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2023-07-14T11:29:54.351 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2023-07-14T11:29:54.351 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2023-07-14T11:29:54.351 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2023-07-14T11:29:54.351 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2023-07-14T11:29:54.352 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2023-07-14T11:29:54.352 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2023-07-14T11:29:54.352 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2023-07-14T11:29:54.352 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2023-07-14T11:29:54.352 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 115, 2023-07-14T11:29:54.353 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2023-07-14T11:29:54.353 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 25, 2023-07-14T11:29:54.353 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 4, 2023-07-14T11:29:54.353 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 6, 2023-07-14T11:29:54.353 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 2, 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 0, 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 2, 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 0 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.354 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2023-07-14T11:29:54.355 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 0, 2023-07-14T11:29:54.355 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:54.355 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2023-07-14T11:29:54.355 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2023-07-14T11:29:54.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.356 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2023-07-14T11:29:54.356 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 0 2023-07-14T11:29:54.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.356 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2023-07-14T11:29:54.356 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:54.357 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:54.357 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2023-07-14T11:29:54.357 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:54.357 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 0, 2023-07-14T11:29:54.357 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:54.358 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 4158, 2023-07-14T11:29:54.359 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 6735386, 2023-07-14T11:29:54.359 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:54.359 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 4158, 2023-07-14T11:29:54.359 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:54.359 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:54.360 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 4158, 2023-07-14T11:29:54.360 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 6735386, 2023-07-14T11:29:54.360 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:54.360 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.360 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.361 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.361 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.361 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:54.361 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:54.362 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:54.363 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 3558, 2023-07-14T11:29:54.363 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 3388358, 2023-07-14T11:29:54.363 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 3558, 2023-07-14T11:29:54.363 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 3388358, 2023-07-14T11:29:54.363 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:54.364 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.364 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.364 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.364 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.365 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:54.365 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:54.365 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2023-07-14T11:29:54.365 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:54.365 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 3558, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 3558, 2023-07-14T11:29:54.366 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 3558, 2023-07-14T11:29:54.367 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 3558, 2023-07-14T11:29:54.367 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:54.367 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.367 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.367 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.368 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.368 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:54.368 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:54.368 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:54.369 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:54.369 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:54.369 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:54.369 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:54.369 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:54.370 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:54.370 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:54.370 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2023-07-14T11:29:54.370 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2023-07-14T11:29:54.370 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:54.371 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x56082c12e460": { 2023-07-14T11:29:54.372 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:54.372 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:54.372 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:54.372 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:54.372 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:54.373 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:54.373 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:54.373 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:54.373 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:54.373 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2023-07-14T11:29:54.374 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2023-07-14T11:29:54.374 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:54.374 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:54.374 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:54.374 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:54.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:54.375 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-07-14T11:29:54.376 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:53 smithi045 ceph-mon[128791]: pgmap v557: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:54.376 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:53 smithi040 ceph-mon[97954]: pgmap v557: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:54.376 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:53 smithi130 ceph-mon[129405]: pgmap v557: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:54.626 INFO:tasks.check_counter:Daemon mds.i mds.exported=0 2023-07-14T11:29:54.626 INFO:tasks.check_counter:Daemon mds.i mds.imported=0 2023-07-14T11:29:54.627 INFO:tasks.check_counter:Daemon mds.i mds.root_rsnaps=0 2023-07-14T11:29:54.627 INFO:tasks.check_counter:Daemon mds.i mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:54.627 INFO:tasks.check_counter:Daemon mds.i mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:54.627 DEBUG:tasks.check_counter:Getting stats from l 2023-07-14T11:29:54.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:3fc0089cc1082313a98853d48b892acdc1385b8b shell --fsid bbe0b002-2235-11ee-9b32-001a4aab830c -- ceph daemon mds.l perf dump 2023-07-14T11:29:55.189 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/mon.c/config 2023-07-14T11:29:55.995 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:55 smithi130 ceph-mon[129405]: pgmap v558: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:56.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:55 smithi040 ceph-mon[97954]: pgmap v558: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:56.072 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:55 smithi045 ceph-mon[128791]: pgmap v558: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:29:56.486 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2023-07-14T11:29:56.487 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-0": { 2023-07-14T11:29:56.487 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 1179, 2023-07-14T11:29:56.487 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 1181, 2023-07-14T11:29:56.487 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 396089, 2023-07-14T11:29:56.488 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 366670, 2023-07-14T11:29:56.488 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 7, 2023-07-14T11:29:56.488 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 2, 2023-07-14T11:29:56.488 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.089270493, 2023-07-14T11:29:56.488 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.024596790, 2023-07-14T11:29:56.489 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.021717625, 2023-07-14T11:29:56.489 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.033676110, 2023-07-14T11:29:56.489 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:56.489 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 1179, 2023-07-14T11:29:56.490 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.036794427, 2023-07-14T11:29:56.490 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000031208 2023-07-14T11:29:56.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.490 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:56.490 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.491 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.491 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.491 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 0, 2023-07-14T11:29:56.491 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 0 2023-07-14T11:29:56.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.492 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-1": { 2023-07-14T11:29:56.492 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 2272, 2023-07-14T11:29:56.492 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 2522, 2023-07-14T11:29:56.492 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 3170927, 2023-07-14T11:29:56.493 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 1283567, 2023-07-14T11:29:56.493 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 7, 2023-07-14T11:29:56.493 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 5, 2023-07-14T11:29:56.493 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.137861917, 2023-07-14T11:29:56.493 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.042981224, 2023-07-14T11:29:56.494 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.033108563, 2023-07-14T11:29:56.494 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.049363175, 2023-07-14T11:29:56.494 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:56.494 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 2520, 2023-07-14T11:29:56.494 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.072692564, 2023-07-14T11:29:56.495 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000028846 2023-07-14T11:29:56.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.495 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:56.495 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.495 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.496 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.496 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.496 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 3264, 2023-07-14T11:29:56.496 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 578784 2023-07-14T11:29:56.496 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: "AsyncMessenger::Worker-2": { 2023-07-14T11:29:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_messages": 598, 2023-07-14T11:29:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages": 364, 2023-07-14T11:29:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_bytes": 2726132, 2023-07-14T11:29:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_bytes": 195888, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_created_connections": 5, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_active_connections": 3, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_total_time": 0.055805703, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_send_time": 0.014388315, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_recv_time": 0.047179283, 2023-07-14T11:29:56.498 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_running_fast_dispatch_time": 0.007085697, 2023-07-14T11:29:56.499 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_messages_queue_lat": { 2023-07-14T11:29:56.499 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 363, 2023-07-14T11:29:56.499 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.012382136, 2023-07-14T11:29:56.499 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000034110 2023-07-14T11:29:56.499 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.500 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_handle_ack_lat": { 2023-07-14T11:29:56.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.500 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.500 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_recv_encrypted_bytes": 2724784, 2023-07-14T11:29:56.501 INFO:teuthology.orchestra.run.smithi130.stdout: "msgr_send_encrypted_bytes": 194944 2023-07-14T11:29:56.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.501 INFO:teuthology.orchestra.run.smithi130.stdout: "cct": { 2023-07-14T11:29:56.501 INFO:teuthology.orchestra.run.smithi130.stdout: "total_workers": 1, 2023-07-14T11:29:56.502 INFO:teuthology.orchestra.run.smithi130.stdout: "unhealthy_workers": 0 2023-07-14T11:29:56.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.502 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-MDSRank": { 2023-07-14T11:29:56.502 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:56.502 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:56.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 5859, 2023-07-14T11:29:56.503 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.131015731, 2023-07-14T11:29:56.503 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000022361 2023-07-14T11:29:56.503 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "finisher-PurgeQueue": { 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "queue_len": 0, 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "complete_latency": { 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.504 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.505 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.505 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.505 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2023-07-14T11:29:56.505 INFO:teuthology.orchestra.run.smithi130.stdout: "request": 0, 2023-07-14T11:29:56.505 INFO:teuthology.orchestra.run.smithi130.stdout: "reply": 0, 2023-07-14T11:29:56.506 INFO:teuthology.orchestra.run.smithi130.stdout: "reply_latency": { 2023-07-14T11:29:56.506 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.506 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.506 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.506 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.507 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_reply": 0, 2023-07-14T11:29:56.507 INFO:teuthology.orchestra.run.smithi130.stdout: "forward": 0, 2023-07-14T11:29:56.508 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_complete": 0, 2023-07-14T11:29:56.508 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_fetch_keys": 0, 2023-07-14T11:29:56.508 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_commit": 0, 2023-07-14T11:29:56.508 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_split": 0, 2023-07-14T11:29:56.509 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_merge": 0, 2023-07-14T11:29:56.509 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes": 676, 2023-07-14T11:29:56.509 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_top": 0, 2023-07-14T11:29:56.509 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_bottom": 0, 2023-07-14T11:29:56.510 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pin_tail": 676, 2023-07-14T11:29:56.510 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_pinned": 676, 2023-07-14T11:29:56.510 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_expired": 16, 2023-07-14T11:29:56.510 INFO:teuthology.orchestra.run.smithi130.stdout: "inodes_with_caps": 0, 2023-07-14T11:29:56.510 INFO:teuthology.orchestra.run.smithi130.stdout: "caps": 0, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "subtrees": 36, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse": 0, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_hit": 0, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_forward": 0, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_discover": 0, 2023-07-14T11:29:56.511 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_dir_fetch": 0, 2023-07-14T11:29:56.512 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_remote_ino": 0, 2023-07-14T11:29:56.512 INFO:teuthology.orchestra.run.smithi130.stdout: "traverse_lock": 0, 2023-07-14T11:29:56.512 INFO:teuthology.orchestra.run.smithi130.stdout: "load_cent": 0, 2023-07-14T11:29:56.512 INFO:teuthology.orchestra.run.smithi130.stdout: "q": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "exported": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "exported_inodes": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "imported": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "imported_inodes": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_dir_fetch": 0, 2023-07-14T11:29:56.513 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_backtrace_fetch": 0, 2023-07-14T11:29:56.514 INFO:teuthology.orchestra.run.smithi130.stdout: "openino_peer_discover": 0, 2023-07-14T11:29:56.514 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rfiles": 0, 2023-07-14T11:29:56.514 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rbytes": 0, 2023-07-14T11:29:56.514 INFO:teuthology.orchestra.run.smithi130.stdout: "root_rsnaps": 0, 2023-07-14T11:29:56.515 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_fetch": 0, 2023-07-14T11:29:56.515 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_set_tag": 0, 2023-07-14T11:29:56.515 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_backtrace_repaired": 0, 2023-07-14T11:29:56.515 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_inotable_repaired": 0, 2023-07-14T11:29:56.516 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_inodes": 0, 2023-07-14T11:29:56.516 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dir_base_inodes": 0, 2023-07-14T11:29:56.516 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_dirfrag_rstats": 0, 2023-07-14T11:29:56.516 INFO:teuthology.orchestra.run.smithi130.stdout: "scrub_file_inodes": 0, 2023-07-14T11:29:56.516 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_inode_file_caps": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_revoke": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_grant": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_trunc": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flushsnap_ack": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_cap_op_flush_ack": 0, 2023-07-14T11:29:56.517 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps": 0, 2023-07-14T11:29:56.518 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_caps_dirty": 0, 2023-07-14T11:29:56.518 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_cap_release": 0, 2023-07-14T11:29:56.518 INFO:teuthology.orchestra.run.smithi130.stdout: "process_request_cap_release": 0 2023-07-14T11:29:56.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.518 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache": { 2023-07-14T11:29:56.519 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update": 0, 2023-07-14T11:29:56.519 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_update_receipt": 0, 2023-07-14T11:29:56.519 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_try_discover": 0, 2023-07-14T11:29:56.519 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_send_discover": 0, 2023-07-14T11:29:56.519 INFO:teuthology.orchestra.run.smithi130.stdout: "dir_handle_discover": 0, 2023-07-14T11:29:56.520 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays": 0, 2023-07-14T11:29:56.520 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_delayed": 0, 2023-07-14T11:29:56.520 INFO:teuthology.orchestra.run.smithi130.stdout: "num_strays_enqueuing": 0, 2023-07-14T11:29:56.520 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_created": 0, 2023-07-14T11:29:56.520 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_enqueued": 0, 2023-07-14T11:29:56.521 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_reintegrated": 0, 2023-07-14T11:29:56.521 INFO:teuthology.orchestra.run.smithi130.stdout: "strays_migrated": 0, 2023-07-14T11:29:56.521 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_processing": 0, 2023-07-14T11:29:56.521 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_enqueued": 0, 2023-07-14T11:29:56.521 INFO:teuthology.orchestra.run.smithi130.stdout: "num_recovering_prioritized": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_started": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "recovery_completed": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_enqueue_scrub": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_exportdir": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_flush": 0, 2023-07-14T11:29:56.522 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragmentdir": 0, 2023-07-14T11:29:56.523 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_fragstats": 0, 2023-07-14T11:29:56.523 INFO:teuthology.orchestra.run.smithi130.stdout: "ireq_inodestats": 0 2023-07-14T11:29:56.523 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.523 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log": { 2023-07-14T11:29:56.523 INFO:teuthology.orchestra.run.smithi130.stdout: "evadd": 0, 2023-07-14T11:29:56.524 INFO:teuthology.orchestra.run.smithi130.stdout: "evex": 0, 2023-07-14T11:29:56.524 INFO:teuthology.orchestra.run.smithi130.stdout: "evtrm": 0, 2023-07-14T11:29:56.524 INFO:teuthology.orchestra.run.smithi130.stdout: "ev": 0, 2023-07-14T11:29:56.524 INFO:teuthology.orchestra.run.smithi130.stdout: "evexg": 0, 2023-07-14T11:29:56.524 INFO:teuthology.orchestra.run.smithi130.stdout: "evexd": 0, 2023-07-14T11:29:56.525 INFO:teuthology.orchestra.run.smithi130.stdout: "segadd": 0, 2023-07-14T11:29:56.525 INFO:teuthology.orchestra.run.smithi130.stdout: "segex": 0, 2023-07-14T11:29:56.525 INFO:teuthology.orchestra.run.smithi130.stdout: "segtrm": 0, 2023-07-14T11:29:56.525 INFO:teuthology.orchestra.run.smithi130.stdout: "seg": 2, 2023-07-14T11:29:56.525 INFO:teuthology.orchestra.run.smithi130.stdout: "segexg": 0, 2023-07-14T11:29:56.526 INFO:teuthology.orchestra.run.smithi130.stdout: "segexd": 0, 2023-07-14T11:29:56.526 INFO:teuthology.orchestra.run.smithi130.stdout: "expos": 4194304, 2023-07-14T11:29:56.526 INFO:teuthology.orchestra.run.smithi130.stdout: "wrpos": 0, 2023-07-14T11:29:56.526 INFO:teuthology.orchestra.run.smithi130.stdout: "rdpos": 6785045, 2023-07-14T11:29:56.526 INFO:teuthology.orchestra.run.smithi130.stdout: "jlat": { 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: "replayed": 1542 2023-07-14T11:29:56.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.528 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mem": { 2023-07-14T11:29:56.528 INFO:teuthology.orchestra.run.smithi130.stdout: "ino": 671, 2023-07-14T11:29:56.528 INFO:teuthology.orchestra.run.smithi130.stdout: "ino+": 898, 2023-07-14T11:29:56.528 INFO:teuthology.orchestra.run.smithi130.stdout: "ino-": 227, 2023-07-14T11:29:56.528 INFO:teuthology.orchestra.run.smithi130.stdout: "dir": 40, 2023-07-14T11:29:56.529 INFO:teuthology.orchestra.run.smithi130.stdout: "dir+": 55, 2023-07-14T11:29:56.529 INFO:teuthology.orchestra.run.smithi130.stdout: "dir-": 15, 2023-07-14T11:29:56.529 INFO:teuthology.orchestra.run.smithi130.stdout: "dn": 676, 2023-07-14T11:29:56.529 INFO:teuthology.orchestra.run.smithi130.stdout: "dn+": 928, 2023-07-14T11:29:56.529 INFO:teuthology.orchestra.run.smithi130.stdout: "dn-": 252, 2023-07-14T11:29:56.530 INFO:teuthology.orchestra.run.smithi130.stdout: "cap": 0, 2023-07-14T11:29:56.530 INFO:teuthology.orchestra.run.smithi130.stdout: "cap+": 0, 2023-07-14T11:29:56.530 INFO:teuthology.orchestra.run.smithi130.stdout: "cap-": 0, 2023-07-14T11:29:56.530 INFO:teuthology.orchestra.run.smithi130.stdout: "rss": 49780, 2023-07-14T11:29:56.530 INFO:teuthology.orchestra.run.smithi130.stdout: "heap": 207156 2023-07-14T11:29:56.531 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.531 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server": { 2023-07-14T11:29:56.531 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_client_request": 0, 2023-07-14T11:29:56.531 INFO:teuthology.orchestra.run.smithi130.stdout: "dispatch_server_request": 0, 2023-07-14T11:29:56.531 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_request": 0, 2023-07-14T11:29:56.532 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_client_session": 0, 2023-07-14T11:29:56.532 INFO:teuthology.orchestra.run.smithi130.stdout: "handle_peer_request": 0, 2023-07-14T11:29:56.532 INFO:teuthology.orchestra.run.smithi130.stdout: "req_create_latency": { 2023-07-14T11:29:56.532 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.532 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.533 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.533 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.533 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getattr_latency": { 2023-07-14T11:29:56.533 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.533 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.534 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.534 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.534 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getfilelock_latency": { 2023-07-14T11:29:56.534 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.534 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: "req_link_latency": { 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.535 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.536 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.536 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookup_latency": { 2023-07-14T11:29:56.536 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.536 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.537 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.537 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookuphash_latency": { 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupino_latency": { 2023-07-14T11:29:56.538 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.539 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.539 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.539 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupname_latency": { 2023-07-14T11:29:56.539 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.540 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.540 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.540 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.540 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupparent_latency": { 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lookupsnap_latency": { 2023-07-14T11:29:56.541 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.542 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.542 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.542 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.542 INFO:teuthology.orchestra.run.smithi130.stdout: "req_lssnap_latency": { 2023-07-14T11:29:56.542 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.543 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.543 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.543 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.543 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mkdir_latency": { 2023-07-14T11:29:56.543 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.544 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.544 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.544 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.544 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mknod_latency": { 2023-07-14T11:29:56.544 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.545 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.545 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.545 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.545 INFO:teuthology.orchestra.run.smithi130.stdout: "req_mksnap_latency": { 2023-07-14T11:29:56.545 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.546 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.546 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.546 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.546 INFO:teuthology.orchestra.run.smithi130.stdout: "req_open_latency": { 2023-07-14T11:29:56.546 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.547 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.547 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.547 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.547 INFO:teuthology.orchestra.run.smithi130.stdout: "req_readdir_latency": { 2023-07-14T11:29:56.547 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rename_latency": { 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.548 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.549 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.549 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.549 INFO:teuthology.orchestra.run.smithi130.stdout: "req_renamesnap_latency": { 2023-07-14T11:29:56.549 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.549 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.550 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.550 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.550 INFO:teuthology.orchestra.run.smithi130.stdout: "req_snapdiff_latency": { 2023-07-14T11:29:56.550 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.550 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.551 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.551 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.551 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmdir_latency": { 2023-07-14T11:29:56.551 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.551 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.552 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.552 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.552 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmsnap_latency": { 2023-07-14T11:29:56.552 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.552 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.553 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.553 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.553 INFO:teuthology.orchestra.run.smithi130.stdout: "req_rmxattr_latency": { 2023-07-14T11:29:56.553 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.553 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setattr_latency": { 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.554 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.555 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.555 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setdirlayout_latency": { 2023-07-14T11:29:56.555 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.555 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.555 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.556 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.556 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setfilelock_latency": { 2023-07-14T11:29:56.556 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.556 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.556 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.557 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setlayout_latency": { 2023-07-14T11:29:56.557 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.557 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.557 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.558 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.558 INFO:teuthology.orchestra.run.smithi130.stdout: "req_setxattr_latency": { 2023-07-14T11:29:56.558 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.558 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.558 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.559 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.559 INFO:teuthology.orchestra.run.smithi130.stdout: "req_symlink_latency": { 2023-07-14T11:29:56.559 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.559 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.559 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.560 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.560 INFO:teuthology.orchestra.run.smithi130.stdout: "req_unlink_latency": { 2023-07-14T11:29:56.560 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.560 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.560 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.561 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_revoke_eviction": 0, 2023-07-14T11:29:56.561 INFO:teuthology.orchestra.run.smithi130.stdout: "cap_acquisition_throttle": 0, 2023-07-14T11:29:56.561 INFO:teuthology.orchestra.run.smithi130.stdout: "req_getvxattr_latency": { 2023-07-14T11:29:56.561 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessions": { 2023-07-14T11:29:56.562 INFO:teuthology.orchestra.run.smithi130.stdout: "session_count": 2, 2023-07-14T11:29:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "session_add": 2, 2023-07-14T11:29:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "session_remove": 0, 2023-07-14T11:29:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_open": 2, 2023-07-14T11:29:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "sessions_stale": 0, 2023-07-14T11:29:56.563 INFO:teuthology.orchestra.run.smithi130.stdout: "total_load": 0, 2023-07-14T11:29:56.564 INFO:teuthology.orchestra.run.smithi130.stdout: "average_load": 0, 2023-07-14T11:29:56.564 INFO:teuthology.orchestra.run.smithi130.stdout: "avg_session_uptime": 3471 2023-07-14T11:29:56.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.564 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool": { 2023-07-14T11:29:56.564 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_bytes": 0, 2023-07-14T11:29:56.565 INFO:teuthology.orchestra.run.smithi130.stdout: "bloom_filter_items": 0, 2023-07-14T11:29:56.565 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_bytes": 0, 2023-07-14T11:29:56.565 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_items": 0, 2023-07-14T11:29:56.565 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_bytes": 0, 2023-07-14T11:29:56.565 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_data_items": 0, 2023-07-14T11:29:56.566 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_bytes": 0, 2023-07-14T11:29:56.566 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_onode_items": 0, 2023-07-14T11:29:56.566 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_bytes": 0, 2023-07-14T11:29:56.566 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_items": 0, 2023-07-14T11:29:56.566 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_bytes": 0, 2023-07-14T11:29:56.567 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_other_items": 0, 2023-07-14T11:29:56.567 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_bytes": 0, 2023-07-14T11:29:56.567 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_buffer_items": 0, 2023-07-14T11:29:56.567 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_bytes": 0, 2023-07-14T11:29:56.567 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_items": 0, 2023-07-14T11:29:56.568 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_bytes": 0, 2023-07-14T11:29:56.568 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blob_items": 0, 2023-07-14T11:29:56.568 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_bytes": 0, 2023-07-14T11:29:56.568 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_shared_blob_items": 0, 2023-07-14T11:29:56.568 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_bytes": 0, 2023-07-14T11:29:56.569 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_inline_bl_items": 0, 2023-07-14T11:29:56.569 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_bytes": 0, 2023-07-14T11:29:56.569 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_items": 0, 2023-07-14T11:29:56.569 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_bytes": 0, 2023-07-14T11:29:56.569 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_txc_items": 0, 2023-07-14T11:29:56.570 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_bytes": 0, 2023-07-14T11:29:56.570 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_deferred_items": 0, 2023-07-14T11:29:56.570 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_bytes": 0, 2023-07-14T11:29:56.570 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_writing_items": 0, 2023-07-14T11:29:56.570 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_bytes": 0, 2023-07-14T11:29:56.571 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_items": 0, 2023-07-14T11:29:56.571 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_bytes": 0, 2023-07-14T11:29:56.571 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_reader_items": 0, 2023-07-14T11:29:56.571 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_bytes": 0, 2023-07-14T11:29:56.571 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_file_writer_items": 0, 2023-07-14T11:29:56.572 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_bytes": 223050, 2023-07-14T11:29:56.572 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_anon_items": 727, 2023-07-14T11:29:56.572 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_bytes": 0, 2023-07-14T11:29:56.572 INFO:teuthology.orchestra.run.smithi130.stdout: "buffer_meta_items": 0, 2023-07-14T11:29:56.572 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bytes": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_items": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_bytes": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mapbl_items": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_bytes": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pglog_items": 0, 2023-07-14T11:29:56.573 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_bytes": 9520, 2023-07-14T11:29:56.574 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_items": 183, 2023-07-14T11:29:56.574 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_bytes": 0, 2023-07-14T11:29:56.574 INFO:teuthology.orchestra.run.smithi130.stdout: "osdmap_mapping_items": 0, 2023-07-14T11:29:56.574 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_bytes": 0, 2023-07-14T11:29:56.574 INFO:teuthology.orchestra.run.smithi130.stdout: "pgmap_items": 0, 2023-07-14T11:29:56.575 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_bytes": 2064556, 2023-07-14T11:29:56.575 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_co_items": 18960, 2023-07-14T11:29:56.575 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_bytes": 0, 2023-07-14T11:29:56.575 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_1_items": 0, 2023-07-14T11:29:56.575 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_bytes": 0, 2023-07-14T11:29:56.576 INFO:teuthology.orchestra.run.smithi130.stdout: "unittest_2_items": 0 2023-07-14T11:29:56.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.576 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter": { 2023-07-14T11:29:56.576 INFO:teuthology.orchestra.run.smithi130.stdout: "op_active": 0, 2023-07-14T11:29:56.576 INFO:teuthology.orchestra.run.smithi130.stdout: "op_laggy": 0, 2023-07-14T11:29:56.577 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send": 3448, 2023-07-14T11:29:56.577 INFO:teuthology.orchestra.run.smithi130.stdout: "op_send_bytes": 21, 2023-07-14T11:29:56.577 INFO:teuthology.orchestra.run.smithi130.stdout: "op_resend": 3, 2023-07-14T11:29:56.577 INFO:teuthology.orchestra.run.smithi130.stdout: "op_reply": 3448, 2023-07-14T11:29:56.577 INFO:teuthology.orchestra.run.smithi130.stdout: "op_latency": { 2023-07-14T11:29:56.578 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3448, 2023-07-14T11:29:56.578 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 2.049962971, 2023-07-14T11:29:56.578 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000594536 2023-07-14T11:29:56.578 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.578 INFO:teuthology.orchestra.run.smithi130.stdout: "op_inflight": 0, 2023-07-14T11:29:56.579 INFO:teuthology.orchestra.run.smithi130.stdout: "oplen_avg": { 2023-07-14T11:29:56.579 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 3448, 2023-07-14T11:29:56.579 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 3449 2023-07-14T11:29:56.579 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.579 INFO:teuthology.orchestra.run.smithi130.stdout: "op": 3448, 2023-07-14T11:29:56.580 INFO:teuthology.orchestra.run.smithi130.stdout: "op_r": 3448, 2023-07-14T11:29:56.580 INFO:teuthology.orchestra.run.smithi130.stdout: "op_w": 0, 2023-07-14T11:29:56.580 INFO:teuthology.orchestra.run.smithi130.stdout: "op_rmw": 0, 2023-07-14T11:29:56.580 INFO:teuthology.orchestra.run.smithi130.stdout: "op_pg": 0, 2023-07-14T11:29:56.580 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_stat": 2205, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_create": 0, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_read": 1241, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_write": 0, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writefull": 0, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_writesame": 0, 2023-07-14T11:29:56.581 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_append": 0, 2023-07-14T11:29:56.582 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_zero": 0, 2023-07-14T11:29:56.582 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_truncate": 0, 2023-07-14T11:29:56.582 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_delete": 0, 2023-07-14T11:29:56.582 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_mapext": 0, 2023-07-14T11:29:56.582 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_sparse_read": 0, 2023-07-14T11:29:56.583 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_clonerange": 0, 2023-07-14T11:29:56.583 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_getxattr": 1, 2023-07-14T11:29:56.583 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_setxattr": 0, 2023-07-14T11:29:56.583 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_cmpxattr": 0, 2023-07-14T11:29:56.583 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_rmxattr": 0, 2023-07-14T11:29:56.584 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_resetxattrs": 0, 2023-07-14T11:29:56.584 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_call": 0, 2023-07-14T11:29:56.584 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_watch": 0, 2023-07-14T11:29:56.584 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_notify": 0, 2023-07-14T11:29:56.584 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_src_cmpxattr": 0, 2023-07-14T11:29:56.585 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls": 0, 2023-07-14T11:29:56.585 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_pgls_filter": 0, 2023-07-14T11:29:56.585 INFO:teuthology.orchestra.run.smithi130.stdout: "osdop_other": 0, 2023-07-14T11:29:56.585 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_active": 0, 2023-07-14T11:29:56.585 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_send": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_resend": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "linger_ping": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_active": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_send": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "poolop_resend": 0, 2023-07-14T11:29:56.586 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_active": 0, 2023-07-14T11:29:56.587 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_send": 0, 2023-07-14T11:29:56.587 INFO:teuthology.orchestra.run.smithi130.stdout: "poolstat_resend": 0, 2023-07-14T11:29:56.587 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_active": 0, 2023-07-14T11:29:56.587 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_send": 0, 2023-07-14T11:29:56.587 INFO:teuthology.orchestra.run.smithi130.stdout: "statfs_resend": 0, 2023-07-14T11:29:56.588 INFO:teuthology.orchestra.run.smithi130.stdout: "command_active": 0, 2023-07-14T11:29:56.588 INFO:teuthology.orchestra.run.smithi130.stdout: "command_send": 0, 2023-07-14T11:29:56.588 INFO:teuthology.orchestra.run.smithi130.stdout: "command_resend": 0, 2023-07-14T11:29:56.588 INFO:teuthology.orchestra.run.smithi130.stdout: "map_epoch": 115, 2023-07-14T11:29:56.588 INFO:teuthology.orchestra.run.smithi130.stdout: "map_full": 0, 2023-07-14T11:29:56.589 INFO:teuthology.orchestra.run.smithi130.stdout: "map_inc": 25, 2023-07-14T11:29:56.589 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_sessions": 6, 2023-07-14T11:29:56.589 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_open": 8, 2023-07-14T11:29:56.589 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_session_close": 2, 2023-07-14T11:29:56.589 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_laggy": 0, 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_wr": 0, 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_rd": 2, 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_del": 0 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: "oft": { 2023-07-14T11:29:56.590 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_objs": 0, 2023-07-14T11:29:56.591 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_kv_pairs": 0, 2023-07-14T11:29:56.591 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_updates": 0, 2023-07-14T11:29:56.591 INFO:teuthology.orchestra.run.smithi130.stdout: "omap_total_removes": 0 2023-07-14T11:29:56.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.591 INFO:teuthology.orchestra.run.smithi130.stdout: "osd-slow-ops": { 2023-07-14T11:29:56.592 INFO:teuthology.orchestra.run.smithi130.stdout: "slow_ops_count": 0 2023-07-14T11:29:56.592 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.592 INFO:teuthology.orchestra.run.smithi130.stdout: "purge_queue": { 2023-07-14T11:29:56.592 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops": 0, 2023-07-14T11:29:56.592 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_ops_high_water": 0, 2023-07-14T11:29:56.593 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing": 0, 2023-07-14T11:29:56.593 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executing_high_water": 0, 2023-07-14T11:29:56.593 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_executed": 0, 2023-07-14T11:29:56.593 INFO:teuthology.orchestra.run.smithi130.stdout: "pq_item_in_journal": 0 2023-07-14T11:29:56.593 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.594 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-msgr_dispatch_throttler-mds": { 2023-07-14T11:29:56.594 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:56.594 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:56.594 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:56.594 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 4045, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 5896300, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 4045, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:56.595 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 4045, 2023-07-14T11:29:56.596 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 5896300, 2023-07-14T11:29:56.596 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:56.596 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.596 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.596 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.597 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.597 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_bytes": { 2023-07-14T11:29:56.597 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:56.597 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 104857600, 2023-07-14T11:29:56.598 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:56.598 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:56.598 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:56.598 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:56.598 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 3448, 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 2591320, 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 3448, 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 2591320, 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:56.599 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.600 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.600 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.601 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.601 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.602 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-objecter_ops": { 2023-07-14T11:29:56.602 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:56.602 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 1024, 2023-07-14T11:29:56.602 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:56.602 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:56.603 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:56.603 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:56.603 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:56.603 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 3448, 2023-07-14T11:29:56.603 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 3448, 2023-07-14T11:29:56.604 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 3448, 2023-07-14T11:29:56.604 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 3448, 2023-07-14T11:29:56.604 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:56.604 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.604 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.605 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.605 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.605 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.605 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle": { 2023-07-14T11:29:56.605 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:56.606 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:56.607 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:56.607 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:56.607 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2023-07-14T11:29:56.607 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2023-07-14T11:29:56.607 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:56.608 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.608 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.608 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.608 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2023-07-14T11:29:56.609 INFO:teuthology.orchestra.run.smithi130.stdout: "throttle-write_buf_throttle-0x5616bb212460": { 2023-07-14T11:29:56.609 INFO:teuthology.orchestra.run.smithi130.stdout: "val": 0, 2023-07-14T11:29:56.609 INFO:teuthology.orchestra.run.smithi130.stdout: "max": 3758096384, 2023-07-14T11:29:56.609 INFO:teuthology.orchestra.run.smithi130.stdout: "get_started": 0, 2023-07-14T11:29:56.609 INFO:teuthology.orchestra.run.smithi130.stdout: "get": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "get_sum": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_fail": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "get_or_fail_success": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "take": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "take_sum": 0, 2023-07-14T11:29:56.610 INFO:teuthology.orchestra.run.smithi130.stdout: "put": 0, 2023-07-14T11:29:56.611 INFO:teuthology.orchestra.run.smithi130.stdout: "put_sum": 0, 2023-07-14T11:29:56.611 INFO:teuthology.orchestra.run.smithi130.stdout: "wait": { 2023-07-14T11:29:56.611 INFO:teuthology.orchestra.run.smithi130.stdout: "avgcount": 0, 2023-07-14T11:29:56.611 INFO:teuthology.orchestra.run.smithi130.stdout: "sum": 0.000000000, 2023-07-14T11:29:56.611 INFO:teuthology.orchestra.run.smithi130.stdout: "avgtime": 0.000000000 2023-07-14T11:29:56.612 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.612 INFO:teuthology.orchestra.run.smithi130.stdout: } 2023-07-14T11:29:56.612 INFO:teuthology.orchestra.run.smithi130.stdout:} 2023-07-14T11:29:56.950 INFO:tasks.check_counter:Daemon mds.l mds.exported=0 2023-07-14T11:29:56.950 INFO:tasks.check_counter:Daemon mds.l mds.imported=0 2023-07-14T11:29:56.950 INFO:tasks.check_counter:Daemon mds.l mds.root_rsnaps=0 2023-07-14T11:29:56.951 INFO:tasks.check_counter:Daemon mds.l mds_server.req_mksnap_latency.avgcount=0 2023-07-14T11:29:56.951 INFO:tasks.check_counter:Daemon mds.l mds_server.req_rmsnap_latency.avgcount=0 2023-07-14T11:29:56.951 DEBUG:teuthology.run_tasks:Unwinding manager kclient 2023-07-14T11:29:56.962 INFO:tasks.kclient:Unmounting kernel clients... 2023-07-14T11:29:56.963 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:29:56.963 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:29:57.025 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:29:57.025 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:29:57.073 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:29:57.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 -- 172.21.15.40:0/1820670774 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e80ffaf0 msgr2=0x7f02e80fff60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.325 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1820670774 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e80ffaf0 0x7f02e80fff60 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7f02d8009990 tx=0x7f02d8009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 -- 172.21.15.40:0/1820670774 shutdown_connections 2023-07-14T11:29:57.326 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1820670774 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e81004a0 0x7f02e810c930 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1820670774 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e8105ed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.327 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1820670774 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e80ffaf0 0x7f02e80fff60 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 -- 172.21.15.40:0/1820670774 >> 172.21.15.40:0/1820670774 conn(0x7f02e80fb630 msgr2=0x7f02e80fda40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:57.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 -- 172.21.15.40:0/1820670774 shutdown_connections 2023-07-14T11:29:57.328 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.321+0000 7f02f07ff700 1 -- 172.21.15.40:0/1820670774 wait complete. 2023-07-14T11:29:57.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.322+0000 7f02f07ff700 1 Processor -- start 2023-07-14T11:29:57.331 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.322+0000 7f02f07ff700 1 -- start start 2023-07-14T11:29:57.332 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.322+0000 7f02f07ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 0x7f02e8197d90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.332 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.322+0000 7f02f07ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 0x7f02e81982d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.332 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02f07ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e819c730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.332 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02f07ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e819cd70 con 0x7f02e80ffaf0 2023-07-14T11:29:57.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02f07ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e819cee0 con 0x7f02e8105b00 2023-07-14T11:29:57.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02f07ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f02e819d0a0 con 0x7f02e81004a0 2023-07-14T11:29:57.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02eed9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e819c730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 0x7f02e81982d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.333 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02ee59b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 0x7f02e8197d90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02eed9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e819c730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53664/0 (socket says 172.21.15.40:53664) 2023-07-14T11:29:57.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02eed9c700 1 -- 172.21.15.40:0/1375712989 learned_addr learned my addr 172.21.15.40:0/1375712989 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:57.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 -- 172.21.15.40:0/1375712989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 msgr2=0x7f02e819c730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e819c730 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.334 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 -- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 msgr2=0x7f02e8197d90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 0x7f02e8197d90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 -- 172.21.15.40:0/1375712989 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f02d80095f0 con 0x7f02e81004a0 2023-07-14T11:29:57.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02ee59b700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 0x7f02e8197d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:57.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.323+0000 7f02edd9a700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 0x7f02e81982d0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f02d8000c00 tx=0x7f02d8005740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:57.335 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.324+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d801d070 con 0x7f02e81004a0 2023-07-14T11:29:57.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.324+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f02d8021410 con 0x7f02e81004a0 2023-07-14T11:29:57.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.324+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f02e819d320 con 0x7f02e81004a0 2023-07-14T11:29:57.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.324+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02d8017650 con 0x7f02e81004a0 2023-07-14T11:29:57.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.324+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02e81a4bc0 con 0x7f02e81004a0 2023-07-14T11:29:57.336 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.325+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f02e8067300 con 0x7f02e81004a0 2023-07-14T11:29:57.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.327+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f02d80217f0 con 0x7f02e81004a0 2023-07-14T11:29:57.337 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.328+0000 7f02df7fe700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 0x7f02d40a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.328+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f02d80ce510 con 0x7f02e81004a0 2023-07-14T11:29:57.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.328+0000 7f02ee59b700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 0x7f02d40a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.329+0000 7f02ee59b700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 0x7f02d40a3830 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f02e4009fd0 tx=0x7f02e4009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:57.338 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.330+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f02d80975c0 con 0x7f02e81004a0 2023-07-14T11:29:57.544 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.541+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f02e81a4d50 con 0x7f02e81004a0 2023-07-14T11:29:57.547 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.543+0000 7f02df7fe700 1 -- 172.21.15.40:0/1375712989 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v115) v1 ==== 74+0+25045 (secure 0 0 0) 0x7f02d809abe0 con 0x7f02e81004a0 2023-07-14T11:29:57.548 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:29:57.549 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":115,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:29:14.678188+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:0/583014137":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/234853232":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/3580297519":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3852248366":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]},{"pool":4,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:29:57.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 msgr2=0x7f02d40a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 0x7f02d40a3830 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f02e4009fd0 tx=0x7f02e4009260 comp rx=0 tx=0).stop 2023-07-14T11:29:57.551 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 msgr2=0x7f02e81982d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 0x7f02e81982d0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7f02d8000c00 tx=0x7f02d8005740 comp rx=0 tx=0).stop 2023-07-14T11:29:57.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 shutdown_connections 2023-07-14T11:29:57.552 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f02d40a1380 0x7f02d40a3830 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f02e8105b00 0x7f02e819c730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.553 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f02e80ffaf0 0x7f02e8197d90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 --2- 172.21.15.40:0/1375712989 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f02e81004a0 0x7f02e81982d0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 >> 172.21.15.40:0/1375712989 conn(0x7f02e80fb630 msgr2=0x7f02e80fbb70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:57.554 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 shutdown_connections 2023-07-14T11:29:57.555 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.546+0000 7f02f07ff700 1 -- 172.21.15.40:0/1375712989 wait complete. 2023-07-14T11:29:57.569 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-07-14T11:05:27.158428+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 12, 'score_stable': 12, 'optimal_score': 0.25, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 2, 'pool_name': 'cephfs_metadata', 'create_time': '2023-07-14T11:09:41.501715+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '83', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}, 'read_balance': {'score_acting': 2.25, 'score_stable': 2.25, 'optimal_score': 1, 'raw_score_acting': 2.25, 'raw_score_stable': 2.25, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 3, 'pool_name': 'cephfs_data', 'create_time': '2023-07-14T11:09:42.564718+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 64, 'pg_placement_num': 64, 'pg_placement_num_target': 64, 'pg_num_target': 64, 'pg_num_pending': 64, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '114', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 29, 'snap_epoch': 114, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.05000000074505806}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}, 'read_balance': {'score_acting': 1.5, 'score_stable': 1.5, 'optimal_score': 1, 'raw_score_acting': 1.5, 'raw_score_stable': 1.5, 'primary_affinity_weighted': 0.9999998807907104, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}, {'pool': 4, 'pool_name': 'cephfs_data_ec', 'create_time': '2023-07-14T11:09:46.118094+0000', 'flags': 8197, 'flags_names': 'hashpspool,ec_overwrites,selfmanaged_snaps', 'type': 3, 'size': 4, 'min_size': 3, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '114', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 29, 'snap_epoch': 114, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'cephfs_data_ec', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 8192, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_min': 64, 'target_size_ratio': 0.8999999761581421}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2023-07-14T11:29:57.570 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2023-07-14T11:29:57.778 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.774+0000 7f3d8afff700 1 -- 172.21.15.40:0/1503561878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 msgr2=0x7f3d8410c920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.774+0000 7f3d8afff700 1 --2- 172.21.15.40:0/1503561878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d8410c920 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7f3d7c00b0a0 tx=0x7f3d7c00b3b0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 -- 172.21.15.40:0/1503561878 shutdown_connections 2023-07-14T11:29:57.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 --2- 172.21.15.40:0/1503561878 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d8410c920 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 --2- 172.21.15.40:0/1503561878 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d84105af0 0x7f3d84105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.779 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 --2- 172.21.15.40:0/1503561878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d840ffae0 0x7f3d840fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 -- 172.21.15.40:0/1503561878 >> 172.21.15.40:0/1503561878 conn(0x7f3d840fb5e0 msgr2=0x7f3d840fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:57.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 -- 172.21.15.40:0/1503561878 shutdown_connections 2023-07-14T11:29:57.780 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.775+0000 7f3d8afff700 1 -- 172.21.15.40:0/1503561878 wait complete. 2023-07-14T11:29:57.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 Processor -- start 2023-07-14T11:29:57.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 -- start start 2023-07-14T11:29:57.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d840ffae0 0x7f3d84197eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.782 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d84105af0 0x7f3d8419c850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d8419ce90 con 0x7f3d84105af0 2023-07-14T11:29:57.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d8419d000 con 0x7f3d840ffae0 2023-07-14T11:29:57.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8afff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d8419d1c0 con 0x7f3d84100490 2023-07-14T11:29:57.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d83fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.783 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d88d9b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d840ffae0 0x7f3d84197eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d8959c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d84105af0 0x7f3d8419c850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d83fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:39658/0 (socket says 172.21.15.40:39658) 2023-07-14T11:29:57.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.776+0000 7f3d83fff700 1 -- 172.21.15.40:0/4293986492 learned_addr learned my addr 172.21.15.40:0/4293986492 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:57.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 -- 172.21.15.40:0/4293986492 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d840ffae0 msgr2=0x7f3d84197eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.784 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d840ffae0 0x7f3d84197eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 -- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d84105af0 msgr2=0x7f3d8419c850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:57.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d84105af0 0x7f3d8419c850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:57.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 -- 172.21.15.40:0/4293986492 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d7c009d00 con 0x7f3d84100490 2023-07-14T11:29:57.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d83fff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f3d7000e9a0 tx=0x7f3d7000ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:57.785 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d70009790 con 0x7f3d84100490 2023-07-14T11:29:57.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f3d70010430 con 0x7f3d84100490 2023-07-14T11:29:57.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d8419d4a0 con 0x7f3d84100490 2023-07-14T11:29:57.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3d70018680 con 0x7f3d84100490 2023-07-14T11:29:57.786 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.777+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d841a4ce0 con 0x7f3d84100490 2023-07-14T11:29:57.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.778+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d84067290 con 0x7f3d84100490 2023-07-14T11:29:57.787 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.781+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f3d700105a0 con 0x7f3d84100490 2023-07-14T11:29:57.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.782+0000 7f3d81ffb700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 0x7f3d740a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:57.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.782+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f3d70014070 con 0x7f3d84100490 2023-07-14T11:29:57.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.782+0000 7f3d88d9b700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 0x7f3d740a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:57.788 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.783+0000 7f3d88d9b700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 0x7f3d740a37f0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f3d78009be0 tx=0x7f3d78009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:57.789 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.784+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f3d70096270 con 0x7f3d84100490 2023-07-14T11:29:57.998 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.994+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f3d841a4fd0 con 0x7f3d84100490 2023-07-14T11:29:57.999 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.995+0000 7f3d81ffb700 1 -- 172.21.15.40:0/4293986492 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v115) v1 ==== 93+0+10 (secure 0 0 0) 0x7f3d70099890 con 0x7f3d84100490 2023-07-14T11:29:57.999 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 1 2023-07-14T11:29:58.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 msgr2=0x7f3d740a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 0x7f3d740a37f0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f3d78009be0 tx=0x7f3d78009260 comp rx=0 tx=0).stop 2023-07-14T11:29:58.001 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 msgr2=0x7f3d841983f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f3d7000e9a0 tx=0x7f3d7000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 shutdown_connections 2023-07-14T11:29:58.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f3d740a1340 0x7f3d740a37f0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d84105af0 0x7f3d8419c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.002 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f3d840ffae0 0x7f3d84197eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 --2- 172.21.15.40:0/4293986492 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f3d84100490 0x7f3d841983f0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 >> 172.21.15.40:0/4293986492 conn(0x7f3d840fb5e0 msgr2=0x7f3d8410b040 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:58.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.997+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 shutdown_connections 2023-07-14T11:29:58.003 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:57.998+0000 7f3d8afff700 1 -- 172.21.15.40:0/4293986492 wait complete. 2023-07-14T11:29:58.020 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_metadata pg_num 2023-07-14T11:29:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 -- 172.21.15.40:0/3508183205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 msgr2=0x7f8de0105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.228 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 --2- 172.21.15.40:0/3508183205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de0105eb0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 crypto rx=0x7f8dd4009990 tx=0x7f8dd4009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 -- 172.21.15.40:0/3508183205 shutdown_connections 2023-07-14T11:29:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 --2- 172.21.15.40:0/3508183205 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de0100480 0x7f8de010c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 --2- 172.21.15.40:0/3508183205 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de0105eb0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 --2- 172.21.15.40:0/3508183205 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de00ffad0 0x7f8de00fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.229 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 -- 172.21.15.40:0/3508183205 >> 172.21.15.40:0/3508183205 conn(0x7f8de00fb5f0 msgr2=0x7f8de00fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 -- 172.21.15.40:0/3508183205 shutdown_connections 2023-07-14T11:29:58.230 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.224+0000 7f8de6dff700 1 -- 172.21.15.40:0/3508183205 wait complete. 2023-07-14T11:29:58.231 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.225+0000 7f8de6dff700 1 Processor -- start 2023-07-14T11:29:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.225+0000 7f8de6dff700 1 -- start start 2023-07-14T11:29:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de0100480 0x7f8de0198320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.232 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de019c780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8de019cdc0 con 0x7f8de0100480 2023-07-14T11:29:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8de019cf30 con 0x7f8de0105ae0 2023-07-14T11:29:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de6dff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8de019d0f0 con 0x7f8de00ffad0 2023-07-14T11:29:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.233 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8ddffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de0100480 0x7f8de0198320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:39678/0 (socket says 172.21.15.40:39678) 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 -- 172.21.15.40:0/1907795973 learned_addr learned my addr 172.21.15.40:0/1907795973 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de539c700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de019c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 -- 172.21.15.40:0/1907795973 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 msgr2=0x7f8de019c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de019c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.234 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 -- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de0100480 msgr2=0x7f8de0198320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de0100480 0x7f8de0198320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.226+0000 7f8de4b9b700 1 -- 172.21.15.40:0/1907795973 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8dd40095f0 con 0x7f8de00ffad0 2023-07-14T11:29:58.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8de4b9b700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f8dd4000c00 tx=0x7f8dd4005720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:58.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dd401d070 con 0x7f8de00ffad0 2023-07-14T11:29:58.235 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8de019d370 con 0x7f8de00ffad0 2023-07-14T11:29:58.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8dd4022470 con 0x7f8de00ffad0 2023-07-14T11:29:58.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8dd4017630 con 0x7f8de00ffad0 2023-07-14T11:29:58.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.227+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8de01a4c10 con 0x7f8de00ffad0 2023-07-14T11:29:58.236 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.228+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8de0067290 con 0x7f8de00ffad0 2023-07-14T11:29:58.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.230+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f8dd400fad0 con 0x7f8de00ffad0 2023-07-14T11:29:58.237 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.232+0000 7f8dddffb700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 0x7f8dd00a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.232+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f8dd40ce490 con 0x7f8de00ffad0 2023-07-14T11:29:58.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.232+0000 7f8ddffff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 0x7f8dd00a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.238 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.232+0000 7f8ddffff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 0x7f8dd00a3770 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f8de01993f0 tx=0x7f8dcc009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:58.239 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.234+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8dd4097540 con 0x7f8de00ffad0 2023-07-14T11:29:58.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:57 smithi130 ceph-mon[129405]: pgmap v559: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:58.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:57 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1375712989' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:58.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:57 smithi040 ceph-mon[97954]: pgmap v559: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:58.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:57 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1375712989' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:58.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:57 smithi045 ceph-mon[128791]: pgmap v559: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:29:58.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:57 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1375712989' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:29:58.446 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.443+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"} v 0) v1 -- 0x7f8de0199130 con 0x7f8de00ffad0 2023-07-14T11:29:58.447 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.443+0000 7f8dddffb700 1 -- 172.21.15.40:0/1907795973 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]=0 v115) v1 ==== 104+0+11 (secure 0 0 0) 0x7f8dd409ab60 con 0x7f8de00ffad0 2023-07-14T11:29:58.447 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:29:58.449 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 msgr2=0x7f8dd00a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 0x7f8dd00a3770 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f8de01993f0 tx=0x7f8dcc009260 comp rx=0 tx=0).stop 2023-07-14T11:29:58.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 msgr2=0x7f8de0197de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.450 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f8dd4000c00 tx=0x7f8dd4005720 comp rx=0 tx=0).stop 2023-07-14T11:29:58.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 shutdown_connections 2023-07-14T11:29:58.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8dd00a12c0 0x7f8dd00a3770 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8de0105ae0 0x7f8de019c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.451 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8de00ffad0 0x7f8de0197de0 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 --2- 172.21.15.40:0/1907795973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8de0100480 0x7f8de0198320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 >> 172.21.15.40:0/1907795973 conn(0x7f8de00fb5f0 msgr2=0x7f8de00fbc40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:58.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 shutdown_connections 2023-07-14T11:29:58.452 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.446+0000 7f8de6dff700 1 -- 172.21.15.40:0/1907795973 wait complete. 2023-07-14T11:29:58.469 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data pg_num 2023-07-14T11:29:58.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 -- 172.21.15.40:0/2101276465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d440ffad0 msgr2=0x7f6d440fff40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 --2- 172.21.15.40:0/2101276465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d440ffad0 0x7f6d440fff40 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f6d38009990 tx=0x7f6d38009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 -- 172.21.15.40:0/2101276465 shutdown_connections 2023-07-14T11:29:58.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 --2- 172.21.15.40:0/2101276465 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d44100480 0x7f6d4410c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 --2- 172.21.15.40:0/2101276465 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44105ae0 0x7f6d44105eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 --2- 172.21.15.40:0/2101276465 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d440ffad0 0x7f6d440fff40 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 -- 172.21.15.40:0/2101276465 >> 172.21.15.40:0/2101276465 conn(0x7f6d440fb5f0 msgr2=0x7f6d440fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:58.680 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 -- 172.21.15.40:0/2101276465 shutdown_connections 2023-07-14T11:29:58.681 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.674+0000 7f6d497ff700 1 -- 172.21.15.40:0/2101276465 wait complete. 2023-07-14T11:29:58.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.675+0000 7f6d497ff700 1 Processor -- start 2023-07-14T11:29:58.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.675+0000 7f6d497ff700 1 -- start start 2023-07-14T11:29:58.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 0x7f6d44072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 0x7f6d441a3db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d44073580 con 0x7f6d44100480 2023-07-14T11:29:58.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d441a42f0 con 0x7f6d440ffad0 2023-07-14T11:29:58.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d497ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6d441a4430 con 0x7f6d44105ae0 2023-07-14T11:29:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d427fc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 0x7f6d44072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d437fe700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 0x7f6d441a3db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53728/0 (socket says 172.21.15.40:53728) 2023-07-14T11:29:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 -- 172.21.15.40:0/49576012 learned_addr learned my addr 172.21.15.40:0/49576012 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:58.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 -- 172.21.15.40:0/49576012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 msgr2=0x7f6d441a3db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 0x7f6d441a3db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 -- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 msgr2=0x7f6d44072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.686 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 0x7f6d44072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d42ffd700 1 -- 172.21.15.40:0/49576012 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6d34009520 con 0x7f6d440ffad0 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d437fe700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 0x7f6d441a3db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.676+0000 7f6d427fc700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 0x7f6d44072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d42ffd700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f6d3400eb30 tx=0x7f6d3400ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d34009880 con 0x7f6d440ffad0 2023-07-14T11:29:58.687 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6d380095f0 con 0x7f6d440ffad0 2023-07-14T11:29:58.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6d340044e0 con 0x7f6d440ffad0 2023-07-14T11:29:58.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6d34003d80 con 0x7f6d440ffad0 2023-07-14T11:29:58.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.677+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6d441a4a30 con 0x7f6d440ffad0 2023-07-14T11:29:58.688 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.678+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6d44067290 con 0x7f6d440ffad0 2023-07-14T11:29:58.689 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.681+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f6d34003f00 con 0x7f6d440ffad0 2023-07-14T11:29:58.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.682+0000 7f6d2bfff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 0x7f6d2c0a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:58.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.682+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f6d34014070 con 0x7f6d440ffad0 2023-07-14T11:29:58.690 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.683+0000 7f6d427fc700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 0x7f6d2c0a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:58.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.683+0000 7f6d427fc700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 0x7f6d2c0a3770 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f6d44074000 tx=0x7f6d38005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:58.691 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.685+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6d34096740 con 0x7f6d440ffad0 2023-07-14T11:29:58.898 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.895+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"} v 0) v1 -- 0x7f6d441a4db0 con 0x7f6d440ffad0 2023-07-14T11:29:58.899 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.896+0000 7f6d2bfff700 1 -- 172.21.15.40:0/49576012 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]=0 v115) v1 ==== 100+0+11 (secure 0 0 0) 0x7f6d34099d60 con 0x7f6d440ffad0 2023-07-14T11:29:58.900 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 64 2023-07-14T11:29:58.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 msgr2=0x7f6d2c0a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 0x7f6d2c0a3770 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f6d44074000 tx=0x7f6d38005fb0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.902 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 msgr2=0x7f6d44072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:58.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 crypto rx=0x7f6d3400eb30 tx=0x7f6d3400ee40 comp rx=0 tx=0).stop 2023-07-14T11:29:58.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 shutdown_connections 2023-07-14T11:29:58.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f6d2c0a12c0 0x7f6d2c0a3770 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6d44105ae0 0x7f6d441a3db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.903 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6d440ffad0 0x7f6d44072810 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 --2- 172.21.15.40:0/49576012 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6d44100480 0x7f6d44072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:58.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 >> 172.21.15.40:0/49576012 conn(0x7f6d440fb5f0 msgr2=0x7f6d441035e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:58.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 shutdown_connections 2023-07-14T11:29:58.904 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:58.898+0000 7f6d497ff700 1 -- 172.21.15.40:0/49576012 wait complete. 2023-07-14T11:29:58.921 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs_data_ec pg_num 2023-07-14T11:29:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/1213995706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38100490 msgr2=0x7f0f3810c920 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.129 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/1213995706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38100490 0x7f0f3810c920 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 crypto rx=0x7f0f30009f90 tx=0x7f0f3000b330 comp rx=0 tx=0).stop 2023-07-14T11:29:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/1213995706 shutdown_connections 2023-07-14T11:29:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/1213995706 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38100490 0x7f0f3810c920 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/1213995706 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38105af0 0x7f0f38105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/1213995706 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f380fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.130 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/1213995706 >> 172.21.15.40:0/1213995706 conn(0x7f0f380fb5e0 msgr2=0x7f0f380fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/1213995706 shutdown_connections 2023-07-14T11:29:59.131 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.125+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/1213995706 wait complete. 2023-07-14T11:29:59.132 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.127+0000 7f0f3e9ff700 1 Processor -- start 2023-07-14T11:29:59.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.127+0000 7f0f3e9ff700 1 -- start start 2023-07-14T11:29:59.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38100490 0x7f0f3819c010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 0x7f0f38195b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f381961b0 con 0x7f0f38100490 2023-07-14T11:29:59.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f38196320 con 0x7f0f380ffae0 2023-07-14T11:29:59.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3e9ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f381964e0 con 0x7f0f38105af0 2023-07-14T11:29:59.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f37fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f37fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53742/0 (socket says 172.21.15.40:53742) 2023-07-14T11:29:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f37fff700 1 -- 172.21.15.40:0/3114541734 learned_addr learned my addr 172.21.15.40:0/3114541734 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 0x7f0f38195b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f377fe700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38100490 0x7f0f3819c010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.135 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 -- 172.21.15.40:0/3114541734 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 msgr2=0x7f0f3819bad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 -- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38100490 msgr2=0x7f0f3819c010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38100490 0x7f0f3819c010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f3cf9c700 1 -- 172.21.15.40:0/3114541734 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f30009c40 con 0x7f0f38105af0 2023-07-14T11:29:59.136 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.128+0000 7f0f37fff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f3cf9c700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 0x7f0f38195b40 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f0f3000b6c0 tx=0x7f0f300094b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f3000e040 con 0x7f0f38105af0 2023-07-14T11:29:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f0f3001d470 con 0x7f0f38105af0 2023-07-14T11:29:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f300052a0 con 0x7f0f38105af0 2023-07-14T11:29:59.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0f38196760 con 0x7f0f38105af0 2023-07-14T11:29:59.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.129+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0f381a2ac0 con 0x7f0f38105af0 2023-07-14T11:29:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.130+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f38067290 con 0x7f0f38105af0 2023-07-14T11:29:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.132+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f0f3001d5e0 con 0x7f0f38105af0 2023-07-14T11:29:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.133+0000 7f0f357fa700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 0x7f0f200a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.133+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f0f300ce330 con 0x7f0f38105af0 2023-07-14T11:29:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.134+0000 7f0f37fff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 0x7f0f200a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.134+0000 7f0f37fff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 0x7f0f200a37f0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f0f28005950 tx=0x7f0f28009310 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:59.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.137+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f0f300973e0 con 0x7f0f38105af0 2023-07-14T11:29:59.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/4293986492' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:58 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1907795973' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/4293986492' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:58 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1907795973' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/4293986492' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:58 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1907795973' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_metadata", "var": "pg_num"}]: dispatch 2023-07-14T11:29:59.351 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.348+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"} v 0) v1 -- 0x7f0f381a2d50 con 0x7f0f38105af0 2023-07-14T11:29:59.352 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.348+0000 7f0f357fa700 1 -- 172.21.15.40:0/3114541734 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]=0 v115) v1 ==== 103+0+11 (secure 0 0 0) 0x7f0f3009aa00 con 0x7f0f38105af0 2023-07-14T11:29:59.353 INFO:teuthology.orchestra.run.smithi040.stdout:pg_num: 32 2023-07-14T11:29:59.354 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 msgr2=0x7f0f200a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 0x7f0f200a37f0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f0f28005950 tx=0x7f0f28009310 comp rx=0 tx=0).stop 2023-07-14T11:29:59.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 msgr2=0x7f0f38195b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 0x7f0f38195b40 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 crypto rx=0x7f0f3000b6c0 tx=0x7f0f300094b0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.355 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 shutdown_connections 2023-07-14T11:29:59.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f0f200a1340 0x7f0f200a37f0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f0f38105af0 0x7f0f38195b40 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f0f380ffae0 0x7f0f3819bad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.356 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 --2- 172.21.15.40:0/3114541734 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0f38100490 0x7f0f3819c010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 >> 172.21.15.40:0/3114541734 conn(0x7f0f380fb5e0 msgr2=0x7f0f381035f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:59.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 shutdown_connections 2023-07-14T11:29:59.357 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.351+0000 7f0f3e9ff700 1 -- 172.21.15.40:0/3114541734 wait complete. 2023-07-14T11:29:59.374 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd blocklist ls 2023-07-14T11:29:59.580 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.576+0000 7f674e3ff700 1 -- 172.21.15.40:0/3139091802 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6748105af0 msgr2=0x7f6748105ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.576+0000 7f674e3ff700 1 --2- 172.21.15.40:0/3139091802 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6748105af0 0x7f6748105ec0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 crypto rx=0x7f6738009990 tx=0x7f6738009ca0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.581 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 -- 172.21.15.40:0/3139091802 shutdown_connections 2023-07-14T11:29:59.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 --2- 172.21.15.40:0/3139091802 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f674810c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 --2- 172.21.15.40:0/3139091802 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6748105af0 0x7f6748105ec0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.582 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 --2- 172.21.15.40:0/3139091802 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f67480ffae0 0x7f67480fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 -- 172.21.15.40:0/3139091802 >> 172.21.15.40:0/3139091802 conn(0x7f67480fb5e0 msgr2=0x7f67480fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:59.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 -- 172.21.15.40:0/3139091802 shutdown_connections 2023-07-14T11:29:59.583 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.577+0000 7f674e3ff700 1 -- 172.21.15.40:0/3139091802 wait complete. 2023-07-14T11:29:59.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 Processor -- start 2023-07-14T11:29:59.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 -- start start 2023-07-14T11:29:59.585 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f6748198340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 0x7f674819c7a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.586 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f674819cde0 con 0x7f6748100490 2023-07-14T11:29:59.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f674819cf50 con 0x7f6748105af0 2023-07-14T11:29:59.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.578+0000 7f674e3ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f674819d110 con 0x7f67480ffae0 2023-07-14T11:29:59.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f674c99c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 0x7f674819c7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.587 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:39712/0 (socket says 172.21.15.40:39712) 2023-07-14T11:29:59.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 -- 172.21.15.40:0/2396414841 learned_addr learned my addr 172.21.15.40:0/2396414841 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:29:59.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f67477fe700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f6748198340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 -- 172.21.15.40:0/2396414841 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 msgr2=0x7f674819c7a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.588 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 0x7f674819c7a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 -- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 msgr2=0x7f6748198340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f6748198340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 -- 172.21.15.40:0/2396414841 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f673c009520 con 0x7f67480ffae0 2023-07-14T11:29:59.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f674c99c700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 0x7f674819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:59.589 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f67477fe700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f6748198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:29:59.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f6747fff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f6738005910 tx=0x7f6738004710 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:59.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f673801d070 con 0x7f67480ffae0 2023-07-14T11:29:59.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6738022470 con 0x7f67480ffae0 2023-07-14T11:29:59.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.579+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6740009c40 con 0x7f67480ffae0 2023-07-14T11:29:59.590 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.580+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6738017650 con 0x7f67480ffae0 2023-07-14T11:29:59.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.580+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f67380095f0 con 0x7f67480ffae0 2023-07-14T11:29:59.591 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.581+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6748067290 con 0x7f67480ffae0 2023-07-14T11:29:59.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.583+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f67380225e0 con 0x7f67480ffae0 2023-07-14T11:29:59.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.584+0000 7f67457fa700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 0x7f67300a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:29:59.592 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.584+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f67380ce320 con 0x7f67480ffae0 2023-07-14T11:29:59.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.584+0000 7f67477fe700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 0x7f67300a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:29:59.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.585+0000 7f67477fe700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 0x7f67300a3770 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f673c009fd0 tx=0x7f673c009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:29:59.593 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.586+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f67380973d0 con 0x7f67480ffae0 2023-07-14T11:29:59.822 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.818+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "osd blocklist ls"} v 0) v1 -- 0x7f6748199150 con 0x7f67480ffae0 2023-07-14T11:29:59.823 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.819+0000 7f67457fa700 1 -- 172.21.15.40:0/2396414841 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "osd blocklist ls"}]=0 listed 16 entries v115) v1 ==== 81+0+941 (secure 0 0 0) 0x7f67380273c0 con 0x7f67480ffae0 2023-07-14T11:29:59.823 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/583014137 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.823 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/234853232 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.824 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/2087222828 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.824 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:29:59.824 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3160212490 2023-07-15T11:03:06.541389+0000 2023-07-14T11:29:59.824 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/4083556243 2023-07-15T11:03:06.541389+0000 2023-07-14T11:29:59.825 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/2326174215 2023-07-15T11:02:39.441675+0000 2023-07-14T11:29:59.825 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3580297519 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.825 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/1897779716 2023-07-15T11:03:06.541389+0000 2023-07-14T11:29:59.825 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3852248366 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/3378800182 2023-07-15T11:02:39.441675+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/2087222828 2023-07-15T11:12:08.501941+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6801/137331595 2023-07-15T11:02:39.441675+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:6800/2376955220 2023-07-15T11:03:06.541389+0000 2023-07-14T11:29:59.826 INFO:teuthology.orchestra.run.smithi040.stdout:172.21.15.40:0/500170929 2023-07-15T11:02:39.441675+0000 2023-07-14T11:29:59.827 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 msgr2=0x7f67300a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 0x7f67300a3770 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f673c009fd0 tx=0x7f673c009260 comp rx=0 tx=0).stop 2023-07-14T11:29:59.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 msgr2=0x7f6748197e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:29:59.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 crypto rx=0x7f6738005910 tx=0x7f6738004710 comp rx=0 tx=0).stop 2023-07-14T11:29:59.828 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 shutdown_connections 2023-07-14T11:29:59.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f67300a12c0 0x7f67300a3770 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6748105af0 0x7f674819c7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f67480ffae0 0x7f6748197e00 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.829 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 --2- 172.21.15.40:0/2396414841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6748100490 0x7f6748198340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:29:59.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 >> 172.21.15.40:0/2396414841 conn(0x7f67480fb5e0 msgr2=0x7f674810ad70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:29:59.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 shutdown_connections 2023-07-14T11:29:59.830 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:29:59.821+0000 7f674e3ff700 1 -- 172.21.15.40:0/2396414841 wait complete. 2023-07-14T11:29:59.830 INFO:teuthology.orchestra.run.smithi040.stderr:listed 16 entries 2023-07-14T11:29:59.844 DEBUG:tasks.cephfs.kernel_mount:Unmounting client client.0... 2023-07-14T11:29:59.844 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:29:59.845 DEBUG:teuthology.orchestra.run.smithi040:> sudo umount /tmp/cephfs 2023-07-14T11:29:59.985 INFO:tasks.cephfs.mount:Cleaning up mount ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:29:59.986 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:29:59.987 DEBUG:teuthology.orchestra.run.smithi040:> (cd /home/ubuntu/cephtest && exec rmdir -- /tmp/cephfs) 2023-07-14T11:30:00.134 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:30:00.135 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list 2023-07-14T11:30:00.201 INFO:teuthology.orchestra.run.smithi040.stdout:ceph-ns--tmp-cephfs (id: 0) 2023-07-14T11:30:00.203 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:30:00.203 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip netns delete ceph-ns--tmp-cephfs 2023-07-14T11:30:00.260 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:59 smithi040 ceph-mon[97954]: pgmap v560: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:30:00.260 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:59 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/49576012' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.260 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:59 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3114541734' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.261 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:29:59 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2396414841' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:30:00.283 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:30:00.283 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link delete ceph-brx 2023-07-14T11:30:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:59 smithi130 ceph-mon[129405]: pgmap v560: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:30:00.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:59 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/49576012' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:59 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3114541734' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:29:59 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2396414841' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:30:00.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:59 smithi045 ceph-mon[128791]: pgmap v560: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 4.2 KiB/s rd, 8 op/s 2023-07-14T11:30:00.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:59 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/49576012' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:59 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3114541734' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs_data_ec", "var": "pg_num"}]: dispatch 2023-07-14T11:30:00.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:29:59 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2396414841' entity='client.admin' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2023-07-14T11:30:00.402 INFO:tasks.kclient:Unmounting kernel clients... 2023-07-14T11:30:00.402 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-07-14T11:30:00.402 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/proc/self/mounts of=/dev/stdout 2023-07-14T11:30:00.457 DEBUG:tasks.cephfs.mount:not mounted; /proc/self/mounts is: sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=16348708k,nr_inodes=4087177,mode=755 0 0 securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0 cgroup /sys/fs/cgroup/systemd cgroup rw,seclabel,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,seclabel,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 cgroup /sys/fs/cgroup/perf_event cgroup rw,seclabel,nosuid,nodev,noexec,relatime,perf_event 0 0 cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,seclabel,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0 cgroup /sys/fs/cgroup/blkio cgroup rw,seclabel,nosuid,nodev,noexec,relatime,blkio 0 0 cgroup /sys/fs/cgroup/memory cgroup rw,seclabel,nosuid,nodev,noexec,relatime,memory 0 0 cgroup /sys/fs/cgroup/pids cgroup rw,seclabel,nosuid,nodev,noexec,relatime,pids 0 0 cgroup /sys/fs/cgroup/cpuset cgroup rw,seclabel,nosuid,nodev,noexec,relatime,cpuset 0 0 cgroup /sys/fs/cgroup/hugetlb cgroup rw,seclabel,nosuid,nodev,noexec,relatime,hugetlb 0 0 cgroup /sys/fs/cgroup/devices cgroup rw,seclabel,nosuid,nodev,noexec,relatime,devices 0 0 cgroup /sys/fs/cgroup/freezer cgroup rw,seclabel,nosuid,nodev,noexec,relatime,freezer 0 0 none /sys/kernel/tracing tracefs rw,seclabel,relatime 0 0 configfs /sys/kernel/config configfs rw,relatime 0 0 /dev/sda1 / ext4 rw,seclabel,relatime 0 0 selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0 mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0 hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime,pagesize=2M 0 0 systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=30,pgrp=1,timeout=0,minproto=5,maxproto=5,direct 0 0 debugfs /sys/kernel/debug debugfs rw,seclabel,relatime 0 0 fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0 sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0 tmpfs /run/user/1000 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3275628k,mode=700,uid=1000,gid=1265 0 0 /dev/mapper/vg_nvme-lv_5 /var/lib/ceph xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota 0 0 /dev/sda1 /var/lib/containers/storage/overlay ext4 rw,seclabel,relatime 0 0 overlay /var/lib/containers/storage/overlay/2ce464d1494fe4a6b7b4dacf48b7588fd3062d78d6149415de5c6d6f30e23445/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/2ce464d1494fe4a6b7b4dacf48b7588fd3062d78d6149415de5c6d6f30e23445/diff,workdir=/var/lib/containers/storage/overlay/2ce464d1494fe4a6b7b4dacf48b7588fd3062d78d6149415de5c6d6f30e23445/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/3ce3c9bdc447598b0064e23f726e55601b67cb5ce70e171f484ecfc4af8e3762/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/3ce3c9bdc447598b0064e23f726e55601b67cb5ce70e171f484ecfc4af8e3762/diff,workdir=/var/lib/containers/storage/overlay/3ce3c9bdc447598b0064e23f726e55601b67cb5ce70e171f484ecfc4af8e3762/work,redirect_dir=on,metacopy=on,volatile 0 0 tmpfs /run/user/0 tmpfs rw,seclabel,nosuid,nodev,relatime,size=3275628k,mode=700 0 0 binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0 overlay /var/lib/containers/storage/overlay/e0b6b41d482c2309d76a2f26c10c110311ce9e48fdd523a047afa48c30d93036/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/e0b6b41d482c2309d76a2f26c10c110311ce9e48fdd523a047afa48c30d93036/diff,workdir=/var/lib/containers/storage/overlay/e0b6b41d482c2309d76a2f26c10c110311ce9e48fdd523a047afa48c30d93036/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/96c0bc92e47064484fd83bdd33afbf207fd9d7f434ed9baa8bd255e3b2cbbe7b/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/96c0bc92e47064484fd83bdd33afbf207fd9d7f434ed9baa8bd255e3b2cbbe7b/diff,workdir=/var/lib/containers/storage/overlay/96c0bc92e47064484fd83bdd33afbf207fd9d7f434ed9baa8bd255e3b2cbbe7b/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/99691aa7a87262c3c02918f812f73a6d553567230eef82952c82878e3ea57d96/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/99691aa7a87262c3c02918f812f73a6d553567230eef82952c82878e3ea57d96/diff,workdir=/var/lib/containers/storage/overlay/99691aa7a87262c3c02918f812f73a6d553567230eef82952c82878e3ea57d96/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/34c9a30fd1bbc91ceb51db43a15f9340e7184054977ea41ee59b7a56070ced09/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/34c9a30fd1bbc91ceb51db43a15f9340e7184054977ea41ee59b7a56070ced09/diff,workdir=/var/lib/containers/storage/overlay/34c9a30fd1bbc91ceb51db43a15f9340e7184054977ea41ee59b7a56070ced09/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/2d3e05c09fedb92aa8c5e9ff5c07910628cb15e9982389974af8a4da9f775b02/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/2d3e05c09fedb92aa8c5e9ff5c07910628cb15e9982389974af8a4da9f775b02/diff,workdir=/var/lib/containers/storage/overlay/2d3e05c09fedb92aa8c5e9ff5c07910628cb15e9982389974af8a4da9f775b02/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/a606b5cb4e1e03c4f0924ea0f21f66bfee26a32b23b81a918895ba27714734fa/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/a606b5cb4e1e03c4f0924ea0f21f66bfee26a32b23b81a918895ba27714734fa/diff,workdir=/var/lib/containers/storage/overlay/a606b5cb4e1e03c4f0924ea0f21f66bfee26a32b23b81a918895ba27714734fa/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/c2f5deeafffe28ac50e7edea59a30324e5da3fcf00fcb6346c8ba6d15ea4bc83/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/c2f5deeafffe28ac50e7edea59a30324e5da3fcf00fcb6346c8ba6d15ea4bc83/diff,workdir=/var/lib/containers/storage/overlay/c2f5deeafffe28ac50e7edea59a30324e5da3fcf00fcb6346c8ba6d15ea4bc83/work,redirect_dir=on,metacopy=on,volatile 0 0 overlay /var/lib/containers/storage/overlay/0f03c4670e8e1d24927bf16460822d30c30e7c5b397c8e0817064cfa9b1456ad/merged overlay rw,context="system_u:object_r:container_file_t:s0:c1022,c1023",nodev,relatime,lowerdir=/var/lib/containers/storage/overlay/l/PQ5FIUT3UG7JY7J6A4XUIBUWFK:/var/lib/containers/storage/overlay/l/DDHKOQYFKGGKOCOXZR6QZJMPKR,upperdir=/var/lib/containers/storage/overlay/0f03c4670e8e1d24927bf16460822d30c30e7c5b397c8e0817064cfa9b1456ad/diff,workdir=/var/lib/containers/storage/overlay/0f03c4670e8e1d24927bf16460822d30c30e7c5b397c8e0817064cfa9b1456ad/work,redirect_dir=on,metacopy=on,volatile 0 0 tmpfs /run/netns tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 2023-07-14T11:30:00.458 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:30:00.458 DEBUG:teuthology.orchestra.run.smithi040:> ip netns list 2023-07-14T11:30:00.511 INFO:teuthology.orchestra.run:Running command with timeout 300 2023-07-14T11:30:00.511 DEBUG:teuthology.orchestra.run.smithi040:> sudo ip link delete ceph-brx 2023-07-14T11:30:00.705 INFO:teuthology.orchestra.run.smithi040.stderr:Cannot find device "ceph-brx" 2023-07-14T11:30:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:30:00.709 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-07-14T11:30:00.720 INFO:tasks.cephadm:Teardown begin 2023-07-14T11:30:00.720 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2023-07-14T11:30:00.721 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:30:00.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.926+0000 7f8e64fff700 1 -- 172.21.15.40:0/3413238392 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e60105b80 msgr2=0x7f8e60105f50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:00.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.926+0000 7f8e64fff700 1 --2- 172.21.15.40:0/3413238392 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e60105b80 0x7f8e60105f50 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 crypto rx=0x7f8e50009990 tx=0x7f8e50009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.926+0000 7f8e64fff700 1 -- 172.21.15.40:0/3413238392 shutdown_connections 2023-07-14T11:30:00.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.926+0000 7f8e64fff700 1 --2- 172.21.15.40:0/3413238392 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e6010c940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.926+0000 7f8e64fff700 1 --2- 172.21.15.40:0/3413238392 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e60105b80 0x7f8e60105f50 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.927+0000 7f8e64fff700 1 --2- 172.21.15.40:0/3413238392 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e600ffb50 0x7f8e600fffc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.927+0000 7f8e64fff700 1 -- 172.21.15.40:0/3413238392 >> 172.21.15.40:0/3413238392 conn(0x7f8e600fb650 msgr2=0x7f8e600fda60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:00.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.927+0000 7f8e64fff700 1 -- 172.21.15.40:0/3413238392 shutdown_connections 2023-07-14T11:30:00.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.927+0000 7f8e64fff700 1 -- 172.21.15.40:0/3413238392 wait complete. 2023-07-14T11:30:00.938 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 Processor -- start 2023-07-14T11:30:00.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 -- start start 2023-07-14T11:30:00.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 0x7f8e60072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:00.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:00.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 0x7f8e601a3cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:00.940 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e60073580 con 0x7f8e60105b80 2023-07-14T11:30:00.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e601a4210 con 0x7f8e60100500 2023-07-14T11:30:00.941 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e64fff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e601a4350 con 0x7f8e600ffb50 2023-07-14T11:30:00.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:00.942 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e5e59c700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 0x7f8e60072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:00.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e57fff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53786/0 (socket says 172.21.15.40:53786) 2023-07-14T11:30:00.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e57fff700 1 -- 172.21.15.40:0/1869974106 learned_addr learned my addr 172.21.15.40:0/1869974106 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:00.943 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.928+0000 7f8e5ed9d700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 0x7f8e601a3cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:00.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 -- 172.21.15.40:0/1869974106 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 msgr2=0x7f8e60072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:00.944 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.945 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 -- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 msgr2=0x7f8e601a3cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:00.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 0x7f8e601a3cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:00.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 -- 172.21.15.40:0/1869974106 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e500095f0 con 0x7f8e600ffb50 2023-07-14T11:30:00.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5ed9d700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 0x7f8e601a3cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:30:00.946 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e57fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:30:00.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e5e59c700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 0x7f8e60072810 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f8e50006010 tx=0x7f8e50005720 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:00.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e5001d070 con 0x7f8e600ffb50 2023-07-14T11:30:00.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f8e50022470 con 0x7f8e600ffb50 2023-07-14T11:30:00.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.929+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e601a4550 con 0x7f8e600ffb50 2023-07-14T11:30:00.947 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.930+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e50017650 con 0x7f8e600ffb50 2023-07-14T11:30:00.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.930+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e601a4970 con 0x7f8e600ffb50 2023-07-14T11:30:00.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.931+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e60067300 con 0x7f8e600ffb50 2023-07-14T11:30:00.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.932+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f8e50022850 con 0x7f8e600ffb50 2023-07-14T11:30:00.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.934+0000 7f8e577fe700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 0x7f8e4c0a37b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:00.948 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.934+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f8e500ce5b0 con 0x7f8e600ffb50 2023-07-14T11:30:00.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.934+0000 7f8e57fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 0x7f8e4c0a37b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:00.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.935+0000 7f8e57fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 0x7f8e4c0a37b0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f8e60074000 tx=0x7f8e48009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:00.949 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:00.936+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f8e50097660 con 0x7f8e600ffb50 2023-07-14T11:30:01.190 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.186+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f8e60073d40 con 0x7f8e600ffb50 2023-07-14T11:30:01.191 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.187+0000 7f8e577fe700 1 -- 172.21.15.40:0/1869974106 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 252 v252) v1 ==== 95+0+10895 (secure 0 0 0) 0x7f8e5009ac80 con 0x7f8e600ffb50 2023-07-14T11:30:01.191 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:30:01.192 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":252,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":252,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:25:24.645879+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":238,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":238,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:30:01.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 msgr2=0x7f8e4c0a37b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 0x7f8e4c0a37b0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f8e60074000 tx=0x7f8e48009260 comp rx=0 tx=0).stop 2023-07-14T11:30:01.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 msgr2=0x7f8e60072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.194 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 0x7f8e60072810 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 crypto rx=0x7f8e50006010 tx=0x7f8e50005720 comp rx=0 tx=0).stop 2023-07-14T11:30:01.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 shutdown_connections 2023-07-14T11:30:01.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f8e4c0a1300 0x7f8e4c0a37b0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e60105b80 0x7f8e601a3cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f8e600ffb50 0x7f8e60072810 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.195 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 --2- 172.21.15.40:0/1869974106 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f8e60100500 0x7f8e60072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 >> 172.21.15.40:0/1869974106 conn(0x7f8e600fb650 msgr2=0x7f8e600fcb30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:01.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 shutdown_connections 2023-07-14T11:30:01.196 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.190+0000 7f8e64fff700 1 -- 172.21.15.40:0/1869974106 wait complete. 2023-07-14T11:30:01.196 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 252 2023-07-14T11:30:01.210 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:00 smithi040 ceph-mon[97954]: overall HEALTH_OK 2023-07-14T11:30:01.213 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2023-07-14T11:30:01.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:00 smithi130 ceph-mon[129405]: overall HEALTH_OK 2023-07-14T11:30:01.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:00 smithi045 ceph-mon[128791]: overall HEALTH_OK 2023-07-14T11:30:01.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 -- 172.21.15.40:0/769140717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 msgr2=0x7f6084105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 --2- 172.21.15.40:0/769140717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084105eb0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 crypto rx=0x7f6080009990 tx=0x7f6080009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 -- 172.21.15.40:0/769140717 shutdown_connections 2023-07-14T11:30:01.418 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 --2- 172.21.15.40:0/769140717 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f6084100480 0x7f608410c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 --2- 172.21.15.40:0/769140717 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084105eb0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 --2- 172.21.15.40:0/769140717 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60840ffad0 0x7f60840fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 -- 172.21.15.40:0/769140717 >> 172.21.15.40:0/769140717 conn(0x7f60840fb590 msgr2=0x7f60840fd9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:01.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 -- 172.21.15.40:0/769140717 shutdown_connections 2023-07-14T11:30:01.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.414+0000 7f608bbff700 1 -- 172.21.15.40:0/769140717 wait complete. 2023-07-14T11:30:01.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.415+0000 7f608bbff700 1 Processor -- start 2023-07-14T11:30:01.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.415+0000 7f608bbff700 1 -- start start 2023-07-14T11:30:01.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.415+0000 7f608bbff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.415+0000 7f608bbff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6084100480 0x7f608419bed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.415+0000 7f608bbff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084195a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608bbff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60841960c0 con 0x7f6084100480 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608bbff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6084196230 con 0x7f60840ffad0 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608bbff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60841963f0 con 0x7f6084105ae0 2023-07-14T11:30:01.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53806/0 (socket says 172.21.15.40:53806) 2023-07-14T11:30:01.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 -- 172.21.15.40:0/3261090792 learned_addr learned my addr 172.21.15.40:0/3261090792 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:01.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608a19c700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084195a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608919a700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6084100480 0x7f608419bed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 -- 172.21.15.40:0/3261090792 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 msgr2=0x7f6084195a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084195a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 -- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6084100480 msgr2=0x7f608419bed0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6084100480 0x7f608419bed0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 -- 172.21.15.40:0/3261090792 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60800095f0 con 0x7f60840ffad0 2023-07-14T11:30:01.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608a19c700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084195a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:30:01.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.416+0000 7f608999b700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f6080005910 tx=0x7f6080005720 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:01.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.417+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f608001d070 con 0x7f60840ffad0 2023-07-14T11:30:01.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.417+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6084196670 con 0x7f60840ffad0 2023-07-14T11:30:01.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.417+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f60841a2980 con 0x7f60840ffad0 2023-07-14T11:30:01.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.417+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f6080022470 con 0x7f60840ffad0 2023-07-14T11:30:01.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.417+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6080017650 con 0x7f60840ffad0 2023-07-14T11:30:01.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.418+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6084067300 con 0x7f60840ffad0 2023-07-14T11:30:01.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.420+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f60800228a0 con 0x7f60840ffad0 2023-07-14T11:30:01.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.421+0000 7f607affd700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 0x7f60700a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.421+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f60800ce1b0 con 0x7f60840ffad0 2023-07-14T11:30:01.427 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.422+0000 7f608919a700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 0x7f60700a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.422+0000 7f608919a700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 0x7f60700a3830 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f6084105260 tx=0x7f6074009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:01.428 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.423+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f6080097260 con 0x7f60840ffad0 2023-07-14T11:30:01.678 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.675+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f6084196f10 con 0x7f60840ffad0 2023-07-14T11:30:01.679 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.676+0000 7f607affd700 1 -- 172.21.15.40:0/3261090792 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 252 v252) v1 ==== 95+0+10895 (secure 0 0 0) 0x7f6080027020 con 0x7f60840ffad0 2023-07-14T11:30:01.680 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:30:01.680 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":252,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":14484,"name":"j","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.40:6835/2807710685","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6834","nonce":2807710685},{"type":"v1","addr":"172.21.15.40:6835","nonce":2807710685}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":4},{"gid":24461,"name":"k","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.45:6833/2674461503","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2674461503},{"type":"v1","addr":"172.21.15.45:6833","nonce":2674461503}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[{"mdsmap":{"epoch":252,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true,"refuse_client_session":false,"refuse_standby_for_another_fs":false},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2023-07-14T11:09:43.127642+0000","modified":"2023-07-14T11:25:24.645879+0000","tableserver":0,"root":0,"session_timeout":300,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":5,"in":[0,1,2,3,4],"up":{"mds_0":24569,"mds_1":24565,"mds_2":24523,"mds_3":24521,"mds_4":24511},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24466":{"gid":24466,"name":"l","rank":4,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6833/2180852091","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":2180852091},{"type":"v1","addr":"172.21.15.130:6833","nonce":2180852091}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24482":{"gid":24482,"name":"d","rank":3,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6837/625677299","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6836","nonce":625677299},{"type":"v1","addr":"172.21.15.40:6837","nonce":625677299}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24494":{"gid":24494,"name":"e","rank":2,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.45:6835/1717726567","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6834","nonce":1717726567},{"type":"v1","addr":"172.21.15.45:6835","nonce":1717726567}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24511":{"gid":24511,"name":"f","rank":4,"incarnation":25,"state":"up:active","state_seq":15,"addr":"172.21.15.130:6835/2228846875","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6834","nonce":2228846875},{"type":"v1","addr":"172.21.15.130:6835","nonce":2228846875}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24521":{"gid":24521,"name":"h","rank":3,"incarnation":23,"state":"up:active","state_seq":14,"addr":"172.21.15.45:6837/3949707739","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6836","nonce":3949707739},{"type":"v1","addr":"172.21.15.45:6837","nonce":3949707739}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24523":{"gid":24523,"name":"c","rank":2,"incarnation":21,"state":"up:active","state_seq":14,"addr":"172.21.15.130:6837/4266211241","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6836","nonce":4266211241},{"type":"v1","addr":"172.21.15.130:6837","nonce":4266211241}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24541":{"gid":24541,"name":"i","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.130:6839/3942717247","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6838","nonce":3942717247},{"type":"v1","addr":"172.21.15.130:6839","nonce":3942717247}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24553":{"gid":24553,"name":"g","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.40:6839/3591632989","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6838","nonce":3591632989},{"type":"v1","addr":"172.21.15.40:6839","nonce":3591632989}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24565":{"gid":24565,"name":"a","rank":1,"incarnation":19,"state":"up:active","state_seq":238,"addr":"172.21.15.40:6841/1989097261","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6840","nonce":1989097261},{"type":"v1","addr":"172.21.15.40:6841","nonce":1989097261}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24569":{"gid":24569,"name":"b","rank":0,"incarnation":14,"state":"up:active","state_seq":238,"addr":"172.21.15.45:6839/180621035","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6838","nonce":180621035},{"type":"v1","addr":"172.21.15.45:6839","nonce":180621035}]},"join_fscid":-1,"export_targets":[],"features":4540138322906710015,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3,4],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0},"id":1}]} 2023-07-14T11:30:01.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 msgr2=0x7f60700a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 0x7f60700a3830 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f6084105260 tx=0x7f6074009260 comp rx=0 tx=0).stop 2023-07-14T11:30:01.682 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 msgr2=0x7f608419b990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 crypto rx=0x7f6080005910 tx=0x7f6080005720 comp rx=0 tx=0).stop 2023-07-14T11:30:01.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 shutdown_connections 2023-07-14T11:30:01.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f60700a1380 0x7f60700a3830 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.678+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f6084105ae0 0x7f6084195a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.683 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.679+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f60840ffad0 0x7f608419b990 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.679+0000 7f608bbff700 1 --2- 172.21.15.40:0/3261090792 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6084100480 0x7f608419bed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.679+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 >> 172.21.15.40:0/3261090792 conn(0x7f60840fb590 msgr2=0x7f60840fbc10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:01.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.679+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 shutdown_connections 2023-07-14T11:30:01.684 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.679+0000 7f608bbff700 1 -- 172.21.15.40:0/3261090792 wait complete. 2023-07-14T11:30:01.685 INFO:teuthology.orchestra.run.smithi040.stderr:dumped fsmap epoch 252 2023-07-14T11:30:01.702 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2023-07-14T11:30:01.908 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 -- 172.21.15.40:0/990898755 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8102950 msgr2=0x7f69c8106910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/990898755 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8102950 0x7f69c8106910 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 crypto rx=0x7f69bc0099c0 tx=0x7f69bc009cd0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 -- 172.21.15.40:0/990898755 shutdown_connections 2023-07-14T11:30:01.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/990898755 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8106ff0 0x7f69c810ebc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/990898755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 0x7f69c8102380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/990898755 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8102950 0x7f69c8106910 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 -- 172.21.15.40:0/990898755 >> 172.21.15.40:0/990898755 conn(0x7f69c80fd800 msgr2=0x7f69c80ffc10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:01.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 -- 172.21.15.40:0/990898755 shutdown_connections 2023-07-14T11:30:01.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.905+0000 7f69ce1ff700 1 -- 172.21.15.40:0/990898755 wait complete. 2023-07-14T11:30:01.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.906+0000 7f69ce1ff700 1 Processor -- start 2023-07-14T11:30:01.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 -- start start 2023-07-14T11:30:01.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 0x7f69c8197ea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8106ff0 0x7f69c819c840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c819ce80 con 0x7f69c8101fb0 2023-07-14T11:30:01.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c819cff0 con 0x7f69c8102950 2023-07-14T11:30:01.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69ce1ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69c819d1b0 con 0x7f69c8106ff0 2023-07-14T11:30:01.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69c6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69c6ffd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53830/0 (socket says 172.21.15.40:53830) 2023-07-14T11:30:01.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69c6ffd700 1 -- 172.21.15.40:0/3222944242 learned_addr learned my addr 172.21.15.40:0/3222944242 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:01.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69c77fe700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 0x7f69c8197ea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.907+0000 7f69c7fff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8106ff0 0x7f69c819c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 -- 172.21.15.40:0/3222944242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8106ff0 msgr2=0x7f69c819c840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8106ff0 0x7f69c819c840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 -- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 msgr2=0x7f69c8197ea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:01.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 0x7f69c8197ea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:01.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 -- 172.21.15.40:0/3222944242 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f69bc0095f0 con 0x7f69c8102950 2023-07-14T11:30:01.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c6ffd700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f69bc0098e0 tx=0x7f69bc00f690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:01.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc01d070 con 0x7f69c8102950 2023-07-14T11:30:01.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69c819d430 con 0x7f69c8102950 2023-07-14T11:30:01.922 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f69bc022470 con 0x7f69c8102950 2023-07-14T11:30:01.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.908+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f69bc017650 con 0x7f69c8102950 2023-07-14T11:30:01.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.909+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69c81a4cd0 con 0x7f69c8102950 2023-07-14T11:30:01.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.910+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f69c8067300 con 0x7f69c8102950 2023-07-14T11:30:01.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.912+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f69bc022850 con 0x7f69c8102950 2023-07-14T11:30:01.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.913+0000 7f69c4ff9700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 0x7f69b00a3830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:01.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.913+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7f69bc0cdff0 con 0x7f69c8102950 2023-07-14T11:30:01.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.913+0000 7f69c77fe700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 0x7f69b00a3830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:01.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.914+0000 7f69c77fe700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 0x7f69b00a3830 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f69b8009c60 tx=0x7f69b8009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:01.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:01.916+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f69bc097200 con 0x7f69c8102950 2023-07-14T11:30:02.046 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:01 smithi040 ceph-mon[97954]: pgmap v561: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:30:02.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:01 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/1869974106' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:01 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3261090792' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.133 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.129+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f69c81a4e60 con 0x7f69c8102950 2023-07-14T11:30:02.134 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.131+0000 7f69c4ff9700 1 -- 172.21.15.40:0/3222944242 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v115) v1 ==== 74+0+25045 (secure 0 0 0) 0x7f69bc09a820 con 0x7f69c8102950 2023-07-14T11:30:02.135 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-07-14T11:30:02.135 INFO:teuthology.orchestra.run.smithi040.stdout:{"epoch":115,"fsid":"bbe0b002-2235-11ee-9b32-001a4aab830c","created":"2023-07-14T11:02:08.998735+0000","modified":"2023-07-14T11:29:14.678188+0000","last_up_change":"2023-07-14T11:08:54.810511+0000","last_in_change":"2023-07-14T11:08:37.829817+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":12,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-07-14T11:05:27.158428+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":12,"score_stable":12,"optimal_score":0.25,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2023-07-14T11:09:41.501715+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2023-07-14T11:09:42.564718+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.05000000074505806},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":0.99999988079071045,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs_data_ec","create_time":"2023-07-14T11:09:46.118094+0000","flags":8197,"flags_names":"hashpspool,ec_overwrites,selfmanaged_snaps","type":3,"size":4,"min_size":3,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"114","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":29,"snap_epoch":114,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"cephfs_data_ec","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"ec960268-07b9-478c-9217-22b4c81f02b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6802","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6803","nonce":2649857957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6804","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6805","nonce":2649857957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6808","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6809","nonce":2649857957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6806","nonce":2649857957},{"type":"v1","addr":"172.21.15.40:6807","nonce":2649857957}]},"public_addr":"172.21.15.40:6803/2649857957","cluster_addr":"172.21.15.40:6805/2649857957","heartbeat_back_addr":"172.21.15.40:6809/2649857957","heartbeat_front_addr":"172.21.15.40:6807/2649857957","state":["exists","up"]},{"osd":1,"uuid":"6d1f5898-462a-4229-8a58-4c31ab9a6b13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6801","nonce":21717094}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6803","nonce":21717094}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6807","nonce":21717094}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":21717094},{"type":"v1","addr":"172.21.15.45:6805","nonce":21717094}]},"public_addr":"172.21.15.45:6801/21717094","cluster_addr":"172.21.15.45:6803/21717094","heartbeat_back_addr":"172.21.15.45:6807/21717094","heartbeat_front_addr":"172.21.15.45:6805/21717094","state":["exists","up"]},{"osd":2,"uuid":"6a098e7e-81d5-4e1d-b613-ff144e76f342","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6801","nonce":1035271032}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6803","nonce":1035271032}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6807","nonce":1035271032}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":1035271032},{"type":"v1","addr":"172.21.15.130:6805","nonce":1035271032}]},"public_addr":"172.21.15.130:6801/1035271032","cluster_addr":"172.21.15.130:6803/1035271032","heartbeat_back_addr":"172.21.15.130:6807/1035271032","heartbeat_front_addr":"172.21.15.130:6805/1035271032","state":["exists","up"]},{"osd":3,"uuid":"d065670e-a11a-4f71-a845-1059f16de6bd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6810","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6811","nonce":741691351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6812","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6813","nonce":741691351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6816","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6817","nonce":741691351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6814","nonce":741691351},{"type":"v1","addr":"172.21.15.40:6815","nonce":741691351}]},"public_addr":"172.21.15.40:6811/741691351","cluster_addr":"172.21.15.40:6813/741691351","heartbeat_back_addr":"172.21.15.40:6817/741691351","heartbeat_front_addr":"172.21.15.40:6815/741691351","state":["exists","up"]},{"osd":4,"uuid":"6fe941ce-8407-48e2-ac92-bedcf87816b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":33,"up_thru":86,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6809","nonce":2253566846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6811","nonce":2253566846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6815","nonce":2253566846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2253566846},{"type":"v1","addr":"172.21.15.45:6813","nonce":2253566846}]},"public_addr":"172.21.15.45:6809/2253566846","cluster_addr":"172.21.15.45:6811/2253566846","heartbeat_back_addr":"172.21.15.45:6815/2253566846","heartbeat_front_addr":"172.21.15.45:6813/2253566846","state":["exists","up"]},{"osd":5,"uuid":"8f6be8ae-e857-4175-b9ea-9da1eeabcfd0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6809","nonce":3826515733}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6811","nonce":3826515733}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6815","nonce":3826515733}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":3826515733},{"type":"v1","addr":"172.21.15.130:6813","nonce":3826515733}]},"public_addr":"172.21.15.130:6809/3826515733","cluster_addr":"172.21.15.130:6811/3826515733","heartbeat_back_addr":"172.21.15.130:6815/3826515733","heartbeat_front_addr":"172.21.15.130:6813/3826515733","state":["exists","up"]},{"osd":6,"uuid":"04fdd37c-4112-429a-b73e-7ff7d0da5177","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6818","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6819","nonce":4118960469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6820","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6821","nonce":4118960469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6824","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6825","nonce":4118960469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6822","nonce":4118960469},{"type":"v1","addr":"172.21.15.40:6823","nonce":4118960469}]},"public_addr":"172.21.15.40:6819/4118960469","cluster_addr":"172.21.15.40:6821/4118960469","heartbeat_back_addr":"172.21.15.40:6825/4118960469","heartbeat_front_addr":"172.21.15.40:6823/4118960469","state":["exists","up"]},{"osd":7,"uuid":"50a6c695-d861-4a14-8473-06d9c6d991d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6817","nonce":2329017282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6819","nonce":2329017282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6823","nonce":2329017282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2329017282},{"type":"v1","addr":"172.21.15.45:6821","nonce":2329017282}]},"public_addr":"172.21.15.45:6817/2329017282","cluster_addr":"172.21.15.45:6819/2329017282","heartbeat_back_addr":"172.21.15.45:6823/2329017282","heartbeat_front_addr":"172.21.15.45:6821/2329017282","state":["exists","up"]},{"osd":8,"uuid":"2fcfa29f-64c0-450c-ba0d-2a74a38550bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":59,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6817","nonce":1753746487}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6819","nonce":1753746487}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6823","nonce":1753746487}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":1753746487},{"type":"v1","addr":"172.21.15.130:6821","nonce":1753746487}]},"public_addr":"172.21.15.130:6817/1753746487","cluster_addr":"172.21.15.130:6819/1753746487","heartbeat_back_addr":"172.21.15.130:6823/1753746487","heartbeat_front_addr":"172.21.15.130:6821/1753746487","state":["exists","up"]},{"osd":9,"uuid":"f6095b5c-ed7d-4f34-a0f3-fc345af6ca0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":67,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6826","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6827","nonce":1901957249}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6828","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6829","nonce":1901957249}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6832","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6833","nonce":1901957249}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:6830","nonce":1901957249},{"type":"v1","addr":"172.21.15.40:6831","nonce":1901957249}]},"public_addr":"172.21.15.40:6827/1901957249","cluster_addr":"172.21.15.40:6829/1901957249","heartbeat_back_addr":"172.21.15.40:6833/1901957249","heartbeat_front_addr":"172.21.15.40:6831/1901957249","state":["exists","up"]},{"osd":10,"uuid":"8fb75cd9-7dc5-4825-b957-ebe7b33c49c7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":72,"up_thru":91,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6825","nonce":3539261425}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6827","nonce":3539261425}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6831","nonce":3539261425}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":3539261425},{"type":"v1","addr":"172.21.15.45:6829","nonce":3539261425}]},"public_addr":"172.21.15.45:6825/3539261425","cluster_addr":"172.21.15.45:6827/3539261425","heartbeat_back_addr":"172.21.15.45:6831/3539261425","heartbeat_front_addr":"172.21.15.45:6829/3539261425","state":["exists","up"]},{"osd":11,"uuid":"ccf993ad-d29e-44b7-9aa1-bf856fc9a92e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":79,"up_thru":89,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6825","nonce":692245935}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6827","nonce":692245935}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6831","nonce":692245935}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":692245935},{"type":"v1","addr":"172.21.15.130:6829","nonce":692245935}]},"public_addr":"172.21.15.130:6825/692245935","cluster_addr":"172.21.15.130:6827/692245935","heartbeat_back_addr":"172.21.15.130:6831/692245935","heartbeat_front_addr":"172.21.15.130:6829/692245935","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:04:41.416275+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:03.914100+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:24.956090+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:05:51.362339+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:14.276546+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:06:35.522308+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:01.510784+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:23.979720+0000","dead_epoch":0},{"osd":8,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:07:44.948309+0000","dead_epoch":0},{"osd":9,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:10.565582+0000","dead_epoch":0},{"osd":10,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:32.609723+0000","dead_epoch":0},{"osd":11,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-07-14T11:08:53.733542+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.0","mappings":[{"from":8,"to":1}]},{"pgid":"2.6","mappings":[{"from":8,"to":11}]},{"pgid":"2.8","mappings":[{"from":2,"to":11}]},{"pgid":"2.e","mappings":[{"from":2,"to":4}]},{"pgid":"3.14","mappings":[{"from":4,"to":6}]},{"pgid":"3.1b","mappings":[{"from":4,"to":8}]},{"pgid":"3.24","mappings":[{"from":4,"to":9}]},{"pgid":"3.25","mappings":[{"from":5,"to":9}]},{"pgid":"3.27","mappings":[{"from":3,"to":9}]},{"pgid":"4.2","mappings":[{"from":9,"to":1}]},{"pgid":"4.12","mappings":[{"from":9,"to":10},{"from":5,"to":2}]},{"pgid":"4.14","mappings":[{"from":9,"to":8}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.40:0/583014137":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/234853232":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3160212490":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/4083556243":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/2326174215":"2023-07-15T11:02:39.441675+0000","172.21.15.40:0/3580297519":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/1897779716":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/3852248366":"2023-07-15T11:12:08.501941+0000","172.21.15.40:0/3378800182":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2087222828":"2023-07-15T11:12:08.501941+0000","172.21.15.40:6801/137331595":"2023-07-15T11:02:39.441675+0000","172.21.15.40:6800/2376955220":"2023-07-15T11:03:06.541389+0000","172.21.15.40:0/500170929":"2023-07-15T11:02:39.441675+0000"},"range_blocklist":{},"erasure_code_profiles":{"cephfs_data_ec":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"2","plugin":"jerasure","technique":"reed_sol_van","w":"8"},"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":3,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]},{"pool":4,"snaps":[{"begin":16,"length":1},{"begin":29,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-07-14T11:30:02.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 msgr2=0x7f69b00a3830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.137 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 0x7f69b00a3830 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f69b8009c60 tx=0x7f69b8009260 comp rx=0 tx=0).stop 2023-07-14T11:30:02.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 msgr2=0x7f69c81983e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 crypto rx=0x7f69bc0098e0 tx=0x7f69bc00f690 comp rx=0 tx=0).stop 2023-07-14T11:30:02.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 shutdown_connections 2023-07-14T11:30:02.138 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f69b00a1380 0x7f69b00a3830 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f69c8106ff0 0x7f69c819c840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69c8101fb0 0x7f69c8197ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.139 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 --2- 172.21.15.40:0/3222944242 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f69c8102950 0x7f69c81983e0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 >> 172.21.15.40:0/3222944242 conn(0x7f69c80fd800 msgr2=0x7f69c80fe200 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:02.140 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.133+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 shutdown_connections 2023-07-14T11:30:02.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.134+0000 7f69ce1ff700 1 -- 172.21.15.40:0/3222944242 wait complete. 2023-07-14T11:30:02.161 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs fail cephfs 2023-07-14T11:30:02.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:01 smithi130 ceph-mon[129405]: pgmap v561: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:30:02.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:01 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/1869974106' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:01 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3261090792' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:01 smithi045 ceph-mon[128791]: pgmap v561: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 6.2 KiB/s rd, 12 op/s 2023-07-14T11:30:02.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:01 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/1869974106' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:01 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3261090792' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2023-07-14T11:30:02.373 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 -- 172.21.15.40:0/4080746647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 msgr2=0x7fab10105eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.373 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 --2- 172.21.15.40:0/4080746647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10105eb0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 crypto rx=0x7fab0c009990 tx=0x7fab0c009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.373 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 -- 172.21.15.40:0/4080746647 shutdown_connections 2023-07-14T11:30:02.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 --2- 172.21.15.40:0/4080746647 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab10100480 0x7fab1010c910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 --2- 172.21.15.40:0/4080746647 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10105eb0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 --2- 172.21.15.40:0/4080746647 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab100ffad0 0x7fab100fff40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 -- 172.21.15.40:0/4080746647 >> 172.21.15.40:0/4080746647 conn(0x7fab100fb5f0 msgr2=0x7fab100fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:02.374 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 -- 172.21.15.40:0/4080746647 shutdown_connections 2023-07-14T11:30:02.375 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.369+0000 7fab187ff700 1 -- 172.21.15.40:0/4080746647 wait complete. 2023-07-14T11:30:02.376 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.370+0000 7fab187ff700 1 Processor -- start 2023-07-14T11:30:02.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 -- start start 2023-07-14T11:30:02.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab100ffad0 0x7fab1019b9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.377 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10195a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.378 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab101960b0 con 0x7fab100ffad0 2023-07-14T11:30:02.378 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab10196220 con 0x7fab10105ae0 2023-07-14T11:30:02.379 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab187ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab101963e0 con 0x7fab10100480 2023-07-14T11:30:02.379 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab15d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:39778/0 (socket says 172.21.15.40:39778) 2023-07-14T11:30:02.380 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 -- 172.21.15.40:0/3631562314 learned_addr learned my addr 172.21.15.40:0/3631562314 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:02.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.371+0000 7fab1659b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab100ffad0 0x7fab1019b9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab16d9c700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10195a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 -- 172.21.15.40:0/3631562314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 msgr2=0x7fab10195a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10195a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 -- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab100ffad0 msgr2=0x7fab1019b9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.382 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab100ffad0 0x7fab1019b9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 -- 172.21.15.40:0/3631562314 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab0c0095f0 con 0x7fab10100480 2023-07-14T11:30:02.383 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab15d9a700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fab0000e9a0 tx=0x7fab0000ecb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:02.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.372+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab00009790 con 0x7fab10100480 2023-07-14T11:30:02.384 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.373+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab101966c0 con 0x7fab10100480 2023-07-14T11:30:02.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.373+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7fab00010430 con 0x7fab10100480 2023-07-14T11:30:02.385 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.373+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fab00018680 con 0x7fab10100480 2023-07-14T11:30:02.386 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.373+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab101a29c0 con 0x7fab10100480 2023-07-14T11:30:02.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.374+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab10067290 con 0x7fab10100480 2023-07-14T11:30:02.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.376+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7fab000105a0 con 0x7fab10100480 2023-07-14T11:30:02.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.377+0000 7fab077fe700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 0x7faafc0a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.388 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.377+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 5 ==== osd_map(115..115 src has 1..115) v4 ==== 9432+0+0 (secure 0 0 0) 0x7fab00014070 con 0x7fab10100480 2023-07-14T11:30:02.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.377+0000 7fab1659b700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 0x7faafc0a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.378+0000 7fab1659b700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 0x7faafc0a37f0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fab0c00b2e0 tx=0x7fab0c005fd0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:02.389 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.379+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7fab000962b0 con 0x7fab10100480 2023-07-14T11:30:02.642 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.639+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_command({"prefix": "fs fail", "fs_name": "cephfs"} v 0) v1 -- 0x7fab101a2ce0 con 0x7fab10100480 2023-07-14T11:30:02.652 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.649+0000 7fab077fe700 1 -- 172.21.15.40:0/3631562314 <== mon.1 v2:172.21.15.130:3300/0 7 ==== mon_command_ack([{"prefix": "fs fail", "fs_name": "cephfs"}]=0 cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. v253) v1 ==== 161+0+0 (secure 0 0 0) 0x7fab000998d0 con 0x7fab10100480 2023-07-14T11:30:02.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 msgr2=0x7faafc0a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 0x7faafc0a37f0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fab0c00b2e0 tx=0x7fab0c005fd0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 msgr2=0x7fab1019bf10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 crypto rx=0x7fab0000e9a0 tx=0x7fab0000ecb0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 shutdown_connections 2023-07-14T11:30:02.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7faafc0a1340 0x7faafc0a37f0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.657 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7fab10105ae0 0x7fab10195a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fab100ffad0 0x7fab1019b9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 --2- 172.21.15.40:0/3631562314 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7fab10100480 0x7fab1019bf10 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.652+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 >> 172.21.15.40:0/3631562314 conn(0x7fab100fb5f0 msgr2=0x7fab101035e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:02.658 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.653+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 shutdown_connections 2023-07-14T11:30:02.659 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.653+0000 7fab187ff700 1 -- 172.21.15.40:0/3631562314 wait complete. 2023-07-14T11:30:02.659 INFO:teuthology.orchestra.run.smithi040.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2023-07-14T11:30:02.687 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs rm cephfs --yes-i-really-mean-it 2023-07-14T11:30:02.898 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a[97930]: 2023-07-14T11:30:02.640+0000 7f266a6bd700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-07-14T11:30:02.899 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:02.899 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:02.900 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mds[132208]: -- 172.21.15.40:0/212689139 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559191e911e0 con 0x559191e66800 2023-07-14T11:30:02.900 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: starting mds.a at 2023-07-14T11:30:02.900 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: 2023-07-14T11:30:02.786+0000 7fa6db05b700 -1 PosixStack listen unable to listen on v2:0.0.0.0:6836/0: (98) Address already in use 2023-07-14T11:30:02.901 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:02.901 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:02.901 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mds[131202]: -- 172.21.15.40:0/1713766409 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5646a163b1e0 con 0x5646a1611000 2023-07-14T11:30:02.901 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: starting mds.d at 2023-07-14T11:30:02.902 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:02.902 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:02.903 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: starting mds.g at 2023-07-14T11:30:02.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 -- 172.21.15.40:0/688791061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 msgr2=0x7f4a700fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 --2- 172.21.15.40:0/688791061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a700fff50 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 crypto rx=0x7f4a64009990 tx=0x7f4a64009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 -- 172.21.15.40:0/688791061 shutdown_connections 2023-07-14T11:30:02.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 --2- 172.21.15.40:0/688791061 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70100490 0x7f4a7010c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 --2- 172.21.15.40:0/688791061 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70105af0 0x7f4a70105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 --2- 172.21.15.40:0/688791061 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a700fff50 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 -- 172.21.15.40:0/688791061 >> 172.21.15.40:0/688791061 conn(0x7f4a700fb5e0 msgr2=0x7f4a700fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:02.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.921+0000 7f4a759ff700 1 -- 172.21.15.40:0/688791061 shutdown_connections 2023-07-14T11:30:02.927 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.922+0000 7f4a759ff700 1 -- 172.21.15.40:0/688791061 wait complete. 2023-07-14T11:30:02.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.923+0000 7f4a759ff700 1 Processor -- start 2023-07-14T11:30:02.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.923+0000 7f4a759ff700 1 -- start start 2023-07-14T11:30:02.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.923+0000 7f4a759ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a70197de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a759ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 0x7f4a70198320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a759ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70105af0 0x7f4a7019c780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a759ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a7019cdc0 con 0x7f4a700ffae0 2023-07-14T11:30:02.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a759ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a7019cf30 con 0x7f4a70100490 2023-07-14T11:30:02.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a759ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a7019d0f0 con 0x7f4a70105af0 2023-07-14T11:30:02.930 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a70197de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 0x7f4a70198320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a70197de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38850/0 (socket says 172.21.15.40:38850) 2023-07-14T11:30:02.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6effd700 1 -- 172.21.15.40:0/2229800687 learned_addr learned my addr 172.21.15.40:0/2229800687 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:02.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6f7fe700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70105af0 0x7f4a7019c780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.931 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 -- 172.21.15.40:0/2229800687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70105af0 msgr2=0x7f4a7019c780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70105af0 0x7f4a7019c780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 -- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 msgr2=0x7f4a70197de0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:02.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a70197de0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:02.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 -- 172.21.15.40:0/2229800687 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a640095f0 con 0x7f4a70100490 2023-07-14T11:30:02.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.924+0000 7f4a6e7fc700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 0x7f4a70198320 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f4a64009990 tx=0x7f4a64009580 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:02.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.925+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a6401d070 con 0x7f4a70100490 2023-07-14T11:30:02.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.925+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4a7019d370 con 0x7f4a70100490 2023-07-14T11:30:02.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.925+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4a64021410 con 0x7f4a70100490 2023-07-14T11:30:02.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.925+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4a64017650 con 0x7f4a70100490 2023-07-14T11:30:02.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.925+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4a701a4c10 con 0x7f4a70100490 2023-07-14T11:30:02.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.926+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4a70067290 con 0x7f4a70100490 2023-07-14T11:30:02.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.928+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f4a640217a0 con 0x7f4a70100490 2023-07-14T11:30:02.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.929+0000 7f4a749fd700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 0x7f4a580a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:02.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.929+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 9774+0+0 (secure 0 0 0) 0x7f4a64021e40 con 0x7f4a70100490 2023-07-14T11:30:02.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.930+0000 7f4a6effd700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 0x7f4a580a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:02.936 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.930+0000 7f4a6effd700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 0x7f4a580a37f0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f4a60009ca0 tx=0x7f4a60009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:02.939 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:02.936+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4a64097bb0 con 0x7f4a70100490 2023-07-14T11:30:03.045 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.045 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.045 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mds[155224]: -- 172.21.15.130:0/2036238557 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ab0ff1d1e0 con 0x55ab0fef2800 2023-07-14T11:30:03.046 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: starting mds.c at 2023-07-14T11:30:03.046 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.046 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.047 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: starting mds.f at 2023-07-14T11:30:03.048 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.048 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.048 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: starting mds.l at 2023-07-14T11:30:03.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3222944242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:30:03.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3631562314' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-07-14T11:30:03.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-07-14T11:30:03.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: osdmap e116: 12 total, 12 up, 12 in 2023-07-14T11:30:03.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2023-07-14T11:30:03.050 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mon[129405]: fsmap cephfs:0/5 2 up:standby, 5 failed 2023-07-14T11:30:03.051 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.051 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.051 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-mds[155567]: -- 172.21.15.130:0/2908541152 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a2490591e0 con 0x55a24902ec00 2023-07-14T11:30:03.051 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:30:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: starting mds.i at 2023-07-14T11:30:03.065 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.065 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.065 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mds[153838]: -- 172.21.15.45:0/1798833237 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55db701511e0 con 0x55db70126800 2023-07-14T11:30:03.066 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: starting mds.h at 2023-07-14T11:30:03.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3222944242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:30:03.066 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3631562314' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-07-14T11:30:03.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-07-14T11:30:03.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: osdmap e116: 12 total, 12 up, 12 in 2023-07-14T11:30:03.067 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2023-07-14T11:30:03.068 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mon[128791]: fsmap cephfs:0/5 2 up:standby, 5 failed 2023-07-14T11:30:03.068 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.069 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.069 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: starting mds.b at 2023-07-14T11:30:03.069 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: ignoring --setuser ceph since I am not root 2023-07-14T11:30:03.070 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: ignoring --setgroup ceph since I am not root 2023-07-14T11:30:03.070 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-mds[153497]: -- 172.21.15.45:0/3498078026 <== mon.1 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dbcb7d71e0 con 0x55dbcb7ad000 2023-07-14T11:30:03.070 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:30:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: starting mds.e at 2023-07-14T11:30:03.186 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3222944242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-07-14T11:30:03.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3631562314' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs fail", "fs_name": "cephfs"}]: dispatch 2023-07-14T11:30:03.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2023-07-14T11:30:03.187 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-07-14T11:30:03.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: osdmap e116: 12 total, 12 up, 12 in 2023-07-14T11:30:03.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2023-07-14T11:30:03.188 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:02 smithi040 ceph-mon[97954]: fsmap cephfs:0/5 2 up:standby, 5 failed 2023-07-14T11:30:03.202 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.198+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} v 0) v1 -- 0x7f4a701a4da0 con 0x7f4a70100490 2023-07-14T11:30:03.656 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.652+0000 7f4a749fd700 1 -- 172.21.15.40:0/2229800687 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]=0 v254) v1 ==== 104+0+0 (secure 0 0 0) 0x7f4a6409b1d0 con 0x7f4a70100490 2023-07-14T11:30:03.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.656+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 msgr2=0x7f4a580a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:03.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.656+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 0x7f4a580a37f0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f4a60009ca0 tx=0x7f4a60009260 comp rx=0 tx=0).stop 2023-07-14T11:30:03.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.656+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 msgr2=0x7f4a70198320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:03.661 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.656+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 0x7f4a70198320 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f4a64009990 tx=0x7f4a64009580 comp rx=0 tx=0).stop 2023-07-14T11:30:03.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 shutdown_connections 2023-07-14T11:30:03.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4a580a1340 0x7f4a580a37f0 secure :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f4a60009ca0 tx=0x7f4a60009260 comp rx=0 tx=0).stop 2023-07-14T11:30:03.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4a70105af0 0x7f4a7019c780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4a700ffae0 0x7f4a70197de0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.662 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 --2- 172.21.15.40:0/2229800687 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4a70100490 0x7f4a70198320 secure :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 crypto rx=0x7f4a64009990 tx=0x7f4a64009580 comp rx=0 tx=0).stop 2023-07-14T11:30:03.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.657+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 >> 172.21.15.40:0/2229800687 conn(0x7f4a700fb5e0 msgr2=0x7f4a7010ada0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:03.663 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.659+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 shutdown_connections 2023-07-14T11:30:03.664 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.660+0000 7f4a759ff700 1 -- 172.21.15.40:0/2229800687 wait complete. 2023-07-14T11:30:03.685 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2023-07-14T11:30:03.909 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 -- 172.21.15.40:0/1331351522 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710105af0 msgr2=0x7f1710105ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:03.910 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 --2- 172.21.15.40:0/1331351522 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710105af0 0x7f1710105ec0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 crypto rx=0x7f1700009990 tx=0x7f1700009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 -- 172.21.15.40:0/1331351522 shutdown_connections 2023-07-14T11:30:03.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 --2- 172.21.15.40:0/1331351522 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710100490 0x7f171010c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 --2- 172.21.15.40:0/1331351522 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710105af0 0x7f1710105ec0 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 --2- 172.21.15.40:0/1331351522 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 0x7f17100fff50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 -- 172.21.15.40:0/1331351522 >> 172.21.15.40:0/1331351522 conn(0x7f17100fb5e0 msgr2=0x7f17100fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:03.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 -- 172.21.15.40:0/1331351522 shutdown_connections 2023-07-14T11:30:03.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.905+0000 7f17165ff700 1 -- 172.21.15.40:0/1331351522 wait complete. 2023-07-14T11:30:03.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.906+0000 7f17165ff700 1 Processor -- start 2023-07-14T11:30:03.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.906+0000 7f17165ff700 1 -- start start 2023-07-14T11:30:03.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 0x7f1710197ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:03.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 0x7f1710198430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:03.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 0x7f171019c840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:03.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f171019ce80 con 0x7f1710100490 2023-07-14T11:30:03.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f171019cff0 con 0x7f1710105af0 2023-07-14T11:30:03.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f17165ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f171019d1b0 con 0x7f17100ffae0 2023-07-14T11:30:03.915 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f170f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 0x7f1710198430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:03.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f1714b9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 0x7f171019c840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:03.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f170f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 0x7f1710198430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:34322/0 (socket says 172.21.15.40:34322) 2023-07-14T11:30:03.916 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f170f7fe700 1 -- 172.21.15.40:0/2398297475 learned_addr learned my addr 172.21.15.40:0/2398297475 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:03.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f170ffff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 0x7f1710197ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:03.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f1714b9c700 1 -- 172.21.15.40:0/2398297475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 msgr2=0x7f1710197ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:03.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.907+0000 7f1714b9c700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 0x7f1710197ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f1714b9c700 1 -- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 msgr2=0x7f1710198430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:03.917 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f1714b9c700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 0x7f1710198430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:03.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f1714b9c700 1 -- 172.21.15.40:0/2398297475 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f17000095f0 con 0x7f1710105af0 2023-07-14T11:30:03.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f1714b9c700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 0x7f171019c840 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f170800efc0 tx=0x7f170800c3a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:03.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1708018070 con 0x7f1710105af0 2023-07-14T11:30:03.918 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f171019d430 con 0x7f1710105af0 2023-07-14T11:30:03.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f170800f040 con 0x7f1710105af0 2023-07-14T11:30:03.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f170800cd40 con 0x7f1710105af0 2023-07-14T11:30:03.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.908+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f17101a4cd0 con 0x7f1710105af0 2023-07-14T11:30:03.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.909+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1710067290 con 0x7f1710105af0 2023-07-14T11:30:03.920 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.911+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f1708003a00 con 0x7f1710105af0 2023-07-14T11:30:03.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.913+0000 7f170d7fa700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 0x7f16f80a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:03.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.913+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(116..116 src has 1..116) v4 ==== 9774+0+0 (secure 0 0 0) 0x7f17080cccd0 con 0x7f1710105af0 2023-07-14T11:30:03.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.913+0000 7f170ffff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 0x7f16f80a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:03.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.914+0000 7f170ffff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 0x7f16f80a37f0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f170000b2e0 tx=0x7f1700005fb0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:03.921 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:03.916+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f1708095c30 con 0x7f1710105af0 2023-07-14T11:30:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: pgmap v563: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:30:04.047 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2229800687' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.048 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-07-14T11:30:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-07-14T11:30:04.049 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: Cluster is now healthy 2023-07-14T11:30:04.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6833/2529486928,v1:172.21.15.130:6835/2529486928] up:boot 2023-07-14T11:30:04.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6835/3933829165,v1:172.21.15.45:6837/3933829165] up:boot 2023-07-14T11:30:04.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6832/3412645795,v1:172.21.15.130:6834/3412645795] up:boot 2023-07-14T11:30:04.050 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6836/2974509004,v1:172.21.15.130:6838/2974509004] up:boot 2023-07-14T11:30:04.051 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6834/3952428526,v1:172.21.15.45:6836/3952428526] up:boot 2023-07-14T11:30:04.051 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.45:6838/2003156225,v1:172.21.15.45:6839/2003156225] up:boot 2023-07-14T11:30:04.051 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.130:6837/1371591966,v1:172.21.15.130:6839/1371591966] up:boot 2023-07-14T11:30:04.052 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6838/3443943742,v1:172.21.15.40:6839/3443943742] up:boot 2023-07-14T11:30:04.053 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6837/2388521418,v1:172.21.15.40:6841/2388521418] up:boot 2023-07-14T11:30:04.053 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: mds.? [v2:172.21.15.40:6836/2899602950,v1:172.21.15.40:6840/2899602950] up:boot 2023-07-14T11:30:04.053 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: fsmap 12 up:standby 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:30:04.054 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:30:04.055 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:30:04.055 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:30:04.055 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:30:04.055 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:30:04.055 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:03 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:30:04.141 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.138+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f17101991f0 con 0x7f1710105af0 2023-07-14T11:30:04.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: pgmap v563: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:30:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2229800687' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-07-14T11:30:04.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-07-14T11:30:04.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: Cluster is now healthy 2023-07-14T11:30:04.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6833/2529486928,v1:172.21.15.130:6835/2529486928] up:boot 2023-07-14T11:30:04.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6835/3933829165,v1:172.21.15.45:6837/3933829165] up:boot 2023-07-14T11:30:04.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6832/3412645795,v1:172.21.15.130:6834/3412645795] up:boot 2023-07-14T11:30:04.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6836/2974509004,v1:172.21.15.130:6838/2974509004] up:boot 2023-07-14T11:30:04.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6834/3952428526,v1:172.21.15.45:6836/3952428526] up:boot 2023-07-14T11:30:04.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.45:6838/2003156225,v1:172.21.15.45:6839/2003156225] up:boot 2023-07-14T11:30:04.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.130:6837/1371591966,v1:172.21.15.130:6839/1371591966] up:boot 2023-07-14T11:30:04.297 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6838/3443943742,v1:172.21.15.40:6839/3443943742] up:boot 2023-07-14T11:30:04.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6837/2388521418,v1:172.21.15.40:6841/2388521418] up:boot 2023-07-14T11:30:04.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: mds.? [v2:172.21.15.40:6836/2899602950,v1:172.21.15.40:6840/2899602950] up:boot 2023-07-14T11:30:04.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:30:04.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: fsmap 12 up:standby 2023-07-14T11:30:04.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:30:04.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:30:04.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:30:04.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:30:04.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:30:04.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:30:04.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:30:04.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:30:04.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:30:04.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:03 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:30:04.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: pgmap v563: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 5.0 KiB/s rd, 9 op/s 2023-07-14T11:30:04.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2229800687' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]: dispatch 2023-07-14T11:30:04.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2023-07-14T11:30:04.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-07-14T11:30:04.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: Cluster is now healthy 2023-07-14T11:30:04.318 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6833/2529486928,v1:172.21.15.130:6835/2529486928] up:boot 2023-07-14T11:30:04.318 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6835/3933829165,v1:172.21.15.45:6837/3933829165] up:boot 2023-07-14T11:30:04.319 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6832/3412645795,v1:172.21.15.130:6834/3412645795] up:boot 2023-07-14T11:30:04.319 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6836/2974509004,v1:172.21.15.130:6838/2974509004] up:boot 2023-07-14T11:30:04.320 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6834/3952428526,v1:172.21.15.45:6836/3952428526] up:boot 2023-07-14T11:30:04.320 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.45:6838/2003156225,v1:172.21.15.45:6839/2003156225] up:boot 2023-07-14T11:30:04.320 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.130:6837/1371591966,v1:172.21.15.130:6839/1371591966] up:boot 2023-07-14T11:30:04.321 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6838/3443943742,v1:172.21.15.40:6839/3443943742] up:boot 2023-07-14T11:30:04.321 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6837/2388521418,v1:172.21.15.40:6841/2388521418] up:boot 2023-07-14T11:30:04.322 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: mds.? [v2:172.21.15.40:6836/2899602950,v1:172.21.15.40:6840/2899602950] up:boot 2023-07-14T11:30:04.322 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2023-07-14T11:30:04.322 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: fsmap 12 up:standby 2023-07-14T11:30:04.323 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "f"}]: dispatch 2023-07-14T11:30:04.323 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "l"}]: dispatch 2023-07-14T11:30:04.324 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "e"}]: dispatch 2023-07-14T11:30:04.324 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "i"}]: dispatch 2023-07-14T11:30:04.324 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "c"}]: dispatch 2023-07-14T11:30:04.325 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "h"}]: dispatch 2023-07-14T11:30:04.325 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "d"}]: dispatch 2023-07-14T11:30:04.325 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2023-07-14T11:30:04.326 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2023-07-14T11:30:04.326 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:03 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "g"}]: dispatch 2023-07-14T11:30:04.906 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.902+0000 7f170d7fa700 1 -- 172.21.15.40:0/2398297475 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_metadata' removed v117) v1 ==== 181+0+0 (secure 0 0 0) 0x7f1708099250 con 0x7f1710105af0 2023-07-14T11:30:04.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.907+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 msgr2=0x7f16f80a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:04.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.907+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 0x7f16f80a37f0 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7f170000b2e0 tx=0x7f1700005fb0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.911 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.907+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 msgr2=0x7f171019c840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:04.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.907+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 0x7f171019c840 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 crypto rx=0x7f170800efc0 tx=0x7f170800c3a0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.912 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 shutdown_connections 2023-07-14T11:30:04.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f16f80a1340 0x7f16f80a37f0 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f1710105af0 0x7f171019c840 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f17100ffae0 0x7f1710197ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.913 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 --2- 172.21.15.40:0/2398297475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1710100490 0x7f1710198430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:04.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.908+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 >> 172.21.15.40:0/2398297475 conn(0x7f17100fb5e0 msgr2=0x7f171010aea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:04.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.910+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 shutdown_connections 2023-07-14T11:30:04.914 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:04.910+0000 7f17165ff700 1 -- 172.21.15.40:0/2398297475 wait complete. 2023-07-14T11:30:04.915 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_metadata' removed 2023-07-14T11:30:04.934 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2023-07-14T11:30:05.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 -- 172.21.15.40:0/712846166 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 msgr2=0x7f4b0c10e6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:05.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 --2- 172.21.15.40:0/712846166 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c10e6f0 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 crypto rx=0x7f4b0400b0a0 tx=0x7f4b0400b3b0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.155 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 -- 172.21.15.40:0/712846166 shutdown_connections 2023-07-14T11:30:05.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 --2- 172.21.15.40:0/712846166 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c10e6f0 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 --2- 172.21.15.40:0/712846166 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c072c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 --2- 172.21.15.40:0/712846166 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c10bdd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 -- 172.21.15.40:0/712846166 >> 172.21.15.40:0/712846166 conn(0x7f4b0c0fb5f0 msgr2=0x7f4b0c0fda00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:05.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 -- 172.21.15.40:0/712846166 shutdown_connections 2023-07-14T11:30:05.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.151+0000 7f4b129ff700 1 -- 172.21.15.40:0/712846166 wait complete. 2023-07-14T11:30:05.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 Processor -- start 2023-07-14T11:30:05.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 -- start start 2023-07-14T11:30:05.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c0ffa30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:05.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c0fff70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:05.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c1004b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:05.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b0c1a85a0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b0c1a8710 con 0x7f4b0c10c310 2023-07-14T11:30:05.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.153+0000 7f4b129ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b0c1a88d0 con 0x7f4b0c0728b0 2023-07-14T11:30:05.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c0fff70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:05.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c0ffa30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:05.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0bfff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c0ffa30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.130:3300/0 says I am v2:172.21.15.40:57632/0 (socket says 172.21.15.40:57632) 2023-07-14T11:30:05.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0bfff700 1 -- 172.21.15.40:0/2534715374 learned_addr learned my addr 172.21.15.40:0/2534715374 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:05.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b10f9c700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c1004b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:05.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 -- 172.21.15.40:0/2534715374 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 msgr2=0x7f4b0c0ffa30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:05.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c0ffa30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 -- 172.21.15.40:0/2534715374 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 msgr2=0x7f4b0c1004b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:05.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c1004b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 -- 172.21.15.40:0/2534715374 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b04009d00 con 0x7f4b0c0731c0 2023-07-14T11:30:05.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.154+0000 7f4b0b7fe700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c0fff70 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f4b0000e9f0 tx=0x7f4b0000ed00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:05.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.155+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b0000c9c0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.155+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b0c1a8bb0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.155+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7f4b00010430 con 0x7f4b0c0731c0 2023-07-14T11:30:05.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.155+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4b00018680 con 0x7f4b0c0731c0 2023-07-14T11:30:05.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.155+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b0c1a90c0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.156+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b0c106c30 con 0x7f4b0c0731c0 2023-07-14T11:30:05.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.158+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7f4b00010860 con 0x7f4b0c0731c0 2023-07-14T11:30:05.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.160+0000 7f4b097fa700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 0x7f4af40a3770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:05.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.160+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(117..117 src has 1..117) v4 ==== 9220+0+0 (secure 0 0 0) 0x7f4b00014070 con 0x7f4b0c0731c0 2023-07-14T11:30:05.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.160+0000 7f4b0bfff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 0x7f4af40a3770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:05.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.161+0000 7f4b0bfff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 0x7f4af40a3770 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f4afc009d20 tx=0x7f4afc009260 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:05.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.164+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7f4b000960b0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:30:05.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2398297475' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:30:05.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:30:05.296 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:04 smithi130 ceph-mon[129405]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:30:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:30:05.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2398297475' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:30:05.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:30:05.299 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:04 smithi040 ceph-mon[97954]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:30:05.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-07-14T11:30:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2398297475' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-07-14T11:30:05.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-07-14T11:30:05.317 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:04 smithi045 ceph-mon[128791]: from='mgr.15582 172.21.15.40:0/4293194947' entity='mgr.x' 2023-07-14T11:30:05.381 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.378+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f4b0c067290 con 0x7f4b0c0731c0 2023-07-14T11:30:05.919 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.915+0000 7f4b097fa700 1 -- 172.21.15.40:0/2534715374 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data' removed v118) v1 ==== 169+0+0 (secure 0 0 0) 0x7f4b000996d0 con 0x7f4b0c0731c0 2023-07-14T11:30:05.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.919+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 msgr2=0x7f4af40a3770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:05.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.919+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 0x7f4af40a3770 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f4afc009d20 tx=0x7f4afc009260 comp rx=0 tx=0).stop 2023-07-14T11:30:05.923 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.919+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 msgr2=0x7f4b0c0fff70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:05.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.919+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c0fff70 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 crypto rx=0x7f4b0000e9f0 tx=0x7f4b0000ed00 comp rx=0 tx=0).stop 2023-07-14T11:30:05.924 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 shutdown_connections 2023-07-14T11:30:05.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7f4af40a12c0 0x7f4af40a3770 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7f4b0c10c310 0x7f4b0c1004b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7f4b0c0728b0 0x7f4b0c0ffa30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.925 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 --2- 172.21.15.40:0/2534715374 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4b0c0731c0 0x7f4b0c0fff70 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:05.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 >> 172.21.15.40:0/2534715374 conn(0x7f4b0c0fb5f0 msgr2=0x7f4b0c103d70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:05.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 shutdown_connections 2023-07-14T11:30:05.926 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:05.920+0000 7f4b129ff700 1 -- 172.21.15.40:0/2534715374 wait complete. 2023-07-14T11:30:05.926 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_data' removed 2023-07-14T11:30:05.944 DEBUG:teuthology.orchestra.run.smithi040:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool rm cephfs_data_ec cephfs_data_ec --yes-i-really-really-mean-it 2023-07-14T11:30:06.156 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 -- 172.21.15.40:0/2446105772 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c0ffae0 msgr2=0x7ff82c0fff50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:06.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 --2- 172.21.15.40:0/2446105772 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c0ffae0 0x7ff82c0fff50 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 crypto rx=0x7ff820009990 tx=0x7ff820009ca0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 -- 172.21.15.40:0/2446105772 shutdown_connections 2023-07-14T11:30:06.157 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 --2- 172.21.15.40:0/2446105772 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c10c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 --2- 172.21.15.40:0/2446105772 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c105af0 0x7ff82c105ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 --2- 172.21.15.40:0/2446105772 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c0ffae0 0x7ff82c0fff50 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 -- 172.21.15.40:0/2446105772 >> 172.21.15.40:0/2446105772 conn(0x7ff82c0fb5e0 msgr2=0x7ff82c0fd9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:06.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 -- 172.21.15.40:0/2446105772 shutdown_connections 2023-07-14T11:30:06.158 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.153+0000 7ff8317ff700 1 -- 172.21.15.40:0/2446105772 wait complete. 2023-07-14T11:30:06.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.154+0000 7ff8317ff700 1 Processor -- start 2023-07-14T11:30:06.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.154+0000 7ff8317ff700 1 -- start start 2023-07-14T11:30:06.160 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:06.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c072d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:06.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 0x7ff82c1a3dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:06.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff82c073580 con 0x7ff82c105af0 2023-07-14T11:30:06.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 -- --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff82c1a4310 con 0x7ff82c0ffae0 2023-07-14T11:30:06.161 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff8317ff700 1 -- --> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff82c1a4450 con 0x7ff82c100490 2023-07-14T11:30:06.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff82b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 0x7ff82c1a3dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:06.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff82affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:06.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff82a7fc700 1 --2- >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c072d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:06.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff82affd700 1 --2- >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.45:3300/0 says I am v2:172.21.15.40:53488/0 (socket says 172.21.15.40:53488) 2023-07-14T11:30:06.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.155+0000 7ff82affd700 1 -- 172.21.15.40:0/3143872782 learned_addr learned my addr 172.21.15.40:0/3143872782 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-07-14T11:30:06.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 -- 172.21.15.40:0/3143872782 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 msgr2=0x7ff82c072d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:06.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c072d50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 -- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 msgr2=0x7ff82c1a3dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:06.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 0x7ff82c1a3dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.163 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 -- 172.21.15.40:0/3143872782 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8200095f0 con 0x7ff82c0ffae0 2023-07-14T11:30:06.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82b7fe700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 0x7ff82c1a3dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:30:06.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82a7fc700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2023-07-14T11:30:06.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff82affd700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7ff81c00e9a0 tx=0x7ff81c00ecb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:06.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff81c009790 con 0x7ff82c0ffae0 2023-07-14T11:30:06.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff82c1a46b0 con 0x7ff82c0ffae0 2023-07-14T11:30:06.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 2 ==== config(26 keys) v1 ==== 1105+0+0 (secure 0 0 0) 0x7ff81c010430 con 0x7ff82c0ffae0 2023-07-14T11:30:06.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff81c018680 con 0x7ff82c0ffae0 2023-07-14T11:30:06.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.156+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff82c1a4be0 con 0x7ff82c0ffae0 2023-07-14T11:30:06.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.157+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff82c067290 con 0x7ff82c0ffae0 2023-07-14T11:30:06.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.159+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 4 ==== mgrmap(e 24) v1 ==== 135846+0+0 (secure 0 0 0) 0x7ff81c0105a0 con 0x7ff82c0ffae0 2023-07-14T11:30:06.166 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.161+0000 7ff813fff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 0x7ff8140a37f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2023-07-14T11:30:06.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.161+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 5 ==== osd_map(118..118 src has 1..118) v4 ==== 8661+0+0 (secure 0 0 0) 0x7ff81c014070 con 0x7ff82c0ffae0 2023-07-14T11:30:06.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.161+0000 7ff82a7fc700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 0x7ff8140a37f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2023-07-14T11:30:06.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.162+0000 7ff82a7fc700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 0x7ff8140a37f0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7ff82c074000 tx=0x7ff820005fd0 comp rx=0 tx=0).ready entity=mgr.15582 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-07-14T11:30:06.167 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.163+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+178038 (secure 0 0 0) 0x7ff81c096420 con 0x7ff82c0ffae0 2023-07-14T11:30:06.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:05 smithi130 ceph-mon[129405]: pgmap v564: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:30:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:05 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:05 smithi130 ceph-mon[129405]: osdmap e117: 12 total, 12 up, 12 in 2023-07-14T11:30:06.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:05 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:06.296 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:05 smithi040 ceph-mon[97954]: pgmap v564: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:30:06.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:05 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:06.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:05 smithi040 ceph-mon[97954]: osdmap e117: 12 total, 12 up, 12 in 2023-07-14T11:30:06.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:05 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:05 smithi045 ceph-mon[128791]: pgmap v564: 129 pgs: 129 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail; 15 KiB/s rd, 54 KiB/s wr, 28 op/s 2023-07-14T11:30:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:05 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:06.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:05 smithi045 ceph-mon[128791]: osdmap e117: 12 total, 12 up, 12 in 2023-07-14T11:30:06.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:05 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:06.379 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.376+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 --> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] -- mon_command({"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7ff82c073d40 con 0x7ff82c0ffae0 2023-07-14T11:30:06.929 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.925+0000 7ff813fff700 1 -- 172.21.15.40:0/3143872782 <== mon.2 v2:172.21.15.45:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]=0 pool 'cephfs_data_ec' removed v119) v1 ==== 178+0+0 (secure 0 0 0) 0x7ff81c026020 con 0x7ff82c0ffae0 2023-07-14T11:30:06.932 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.928+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 msgr2=0x7ff8140a37f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:06.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.928+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 0x7ff8140a37f0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7ff82c074000 tx=0x7ff820005fd0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 msgr2=0x7ff82c072810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-07-14T11:30:06.933 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 crypto rx=0x7ff81c00e9a0 tx=0x7ff81c00ecb0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 shutdown_connections 2023-07-14T11:30:06.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:6800/4243839420,v1:172.21.15.40:6801/4243839420] conn(0x7ff8140a1340 0x7ff8140a37f0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff82c105af0 0x7ff82c1a3dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] conn(0x7ff82c0ffae0 0x7ff82c072810 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.934 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 --2- 172.21.15.40:0/3143872782 >> [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] conn(0x7ff82c100490 0x7ff82c072d50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2023-07-14T11:30:06.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 >> 172.21.15.40:0/3143872782 conn(0x7ff82c0fb5e0 msgr2=0x7ff82c0fbc40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-07-14T11:30:06.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 shutdown_connections 2023-07-14T11:30:06.935 INFO:teuthology.orchestra.run.smithi040.stderr:2023-07-14T11:30:06.929+0000 7ff8317ff700 1 -- 172.21.15.40:0/3143872782 wait complete. 2023-07-14T11:30:06.935 INFO:teuthology.orchestra.run.smithi040.stderr:pool 'cephfs_data_ec' removed 2023-07-14T11:30:06.957 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:30:07.069 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:30:07.212 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:06 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:07.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:06 smithi045 ceph-mon[128791]: osdmap e118: 12 total, 12 up, 12 in 2023-07-14T11:30:07.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:06 smithi045 ceph-mon[128791]: from='client.? 172.21.15.40:0/3143872782' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.213 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:06 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:30:07.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:06 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:07.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:06 smithi130 ceph-mon[129405]: osdmap e118: 12 total, 12 up, 12 in 2023-07-14T11:30:07.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:06 smithi130 ceph-mon[129405]: from='client.? 172.21.15.40:0/3143872782' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.247 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:06 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.252 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-07-14T11:30:07.252 DEBUG:teuthology.orchestra.run.smithi040:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-07-14T11:30:07.296 DEBUG:teuthology.orchestra.run.smithi045:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-07-14T11:30:07.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:06 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/2534715374' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:07.297 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:06 smithi040 ceph-mon[97954]: osdmap e118: 12 total, 12 up, 12 in 2023-07-14T11:30:07.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:06 smithi040 ceph-mon[97954]: from='client.? 172.21.15.40:0/3143872782' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.298 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:06 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]: dispatch 2023-07-14T11:30:07.416 DEBUG:teuthology.orchestra.run.smithi130:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-07-14T11:30:07.437 INFO:tasks.cephadm:Stopping all daemons... 2023-07-14T11:30:07.438 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-07-14T11:30:07.438 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a 2023-07-14T11:30:07.796 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 systemd[1]: Stopping Ceph mon.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:08.208 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 ceph-mon[97954]: pgmap v567: 33 pgs: 33 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:30:08.208 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 ceph-mon[97954]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:08.208 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 ceph-mon[97954]: osdmap e119: 12 total, 12 up, 12 in 2023-07-14T11:30:08.209 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a[97930]: 2023-07-14T11:30:07.976+0000 7f266fec8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:08.209 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a[97930]: 2023-07-14T11:30:07.976+0000 7f266fec8700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-07-14T11:30:08.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:07 smithi130 ceph-mon[129405]: pgmap v567: 33 pgs: 33 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:30:08.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:07 smithi130 ceph-mon[129405]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:08.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:07 smithi130 ceph-mon[129405]: osdmap e119: 12 total, 12 up, 12 in 2023-07-14T11:30:08.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:07 smithi045 ceph-mon[128791]: pgmap v567: 33 pgs: 33 active+clean; 5.5 GiB data, 12 GiB used, 1.0 TiB / 1.0 TiB avail 2023-07-14T11:30:08.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:07 smithi045 ceph-mon[128791]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data_ec", "pool2": "cephfs_data_ec", "yes_i_really_really_mean_it": true}]': finished 2023-07-14T11:30:08.316 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:07 smithi045 ceph-mon[128791]: osdmap e119: 12 total, 12 up, 12 in 2023-07-14T11:30:08.546 INFO:journalctl@ceph.mon.a.smithi040.stdout:Jul 14 11:30:08 smithi040 bash[161087]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-a 2023-07-14T11:30:08.811 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.a.service' 2023-07-14T11:30:08.951 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:08.951 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-07-14T11:30:08.952 INFO:tasks.cephadm.mon.c:Stopping mon.b... 2023-07-14T11:30:08.952 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.b 2023-07-14T11:30:09.315 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:08 smithi045 systemd[1]: Stopping Ceph mon.b for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:10.565 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-b[128767]: 2023-07-14T11:30:10.185+0000 7f651501c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:10.566 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-b[128767]: 2023-07-14T11:30:10.185+0000 7f651501c700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2023-07-14T11:30:11.815 INFO:journalctl@ceph.mon.b.smithi045.stdout:Jul 14 11:30:11 smithi045 bash[160927]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-b 2023-07-14T11:30:13.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.b.service' 2023-07-14T11:30:13.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:13.082 INFO:tasks.cephadm.mon.c:Stopped mon.b 2023-07-14T11:30:13.082 INFO:tasks.cephadm.mon.c:Stopping mon.c... 2023-07-14T11:30:13.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.c 2023-07-14T11:30:13.469 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:13 smithi130 systemd[1]: Stopping Ceph mon.c for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:13.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-c[129381]: 2023-07-14T11:30:13.465+0000 7f85305ae700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:13.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-c[129381]: 2023-07-14T11:30:13.465+0000 7f85305ae700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2023-07-14T11:30:14.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Jul 14 11:30:13 smithi130 bash[162453]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mon-c 2023-07-14T11:30:14.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mon.c.service' 2023-07-14T11:30:14.253 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:14.254 INFO:tasks.cephadm.mon.c:Stopped mon.c 2023-07-14T11:30:14.254 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-07-14T11:30:14.254 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x 2023-07-14T11:30:14.783 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:30:14 smithi040 systemd[1]: Stopping Ceph mgr.x for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:15.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:30:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:30:14.777+0000 7fc78f26d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:15.047 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:30:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x[98202]: 2023-07-14T11:30:14.777+0000 7fc78f26d700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-07-14T11:30:15.546 INFO:journalctl@ceph.mgr.x.smithi040.stdout:Jul 14 11:30:15 smithi040 bash[161240]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-x 2023-07-14T11:30:15.586 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.x.service' 2023-07-14T11:30:15.677 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:15.678 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-07-14T11:30:15.678 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-07-14T11:30:15.679 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.y 2023-07-14T11:30:16.065 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:30:15 smithi045 systemd[1]: Stopping Ceph mgr.y for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:16.446 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:30:16 smithi045 bash[161077]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-y 2023-07-14T11:30:16.447 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:30:16 smithi045 systemd[1]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-07-14T11:30:16.686 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.y.service' 2023-07-14T11:30:16.713 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:30:16 smithi045 systemd[1]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.y.service: Failed with result 'exit-code'. 2023-07-14T11:30:16.714 INFO:journalctl@ceph.mgr.y.smithi045.stdout:Jul 14 11:30:16 smithi045 systemd[1]: Stopped Ceph mgr.y for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:30:16.726 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:16.727 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-07-14T11:30:16.727 INFO:tasks.cephadm.mgr.z:Stopping mgr.z... 2023-07-14T11:30:16.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.z 2023-07-14T11:30:17.043 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:30:16 smithi130 systemd[1]: Stopping Ceph mgr.z for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:17.516 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:30:17 smithi130 bash[162604]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mgr-z 2023-07-14T11:30:17.517 INFO:journalctl@ceph.mgr.z.smithi130.stdout:Jul 14 11:30:17 smithi130 systemd[1]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.z.service: Main process exited, code=exited, status=143/n/a 2023-07-14T11:30:17.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mgr.z.service' 2023-07-14T11:30:17.561 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:17.562 INFO:tasks.cephadm.mgr.z:Stopped mgr.z 2023-07-14T11:30:17.562 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-07-14T11:30:17.563 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.0 2023-07-14T11:30:18.047 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:30:17 smithi040 systemd[1]: Stopping Ceph osd.0 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:18.547 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:30:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0[110217]: 2023-07-14T11:30:18.077+0000 7f428f7d2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:18.547 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:30:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0[110217]: 2023-07-14T11:30:18.077+0000 7f428f7d2700 -1 osd.0 119 *** Got signal Terminated *** 2023-07-14T11:30:18.547 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:30:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0[110217]: 2023-07-14T11:30:18.077+0000 7f428f7d2700 -1 osd.0 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:24.297 INFO:journalctl@ceph.osd.0.smithi040.stdout:Jul 14 11:30:24 smithi040 bash[161391]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-0 2023-07-14T11:30:26.640 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.0.service' 2023-07-14T11:30:26.731 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:26.732 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-07-14T11:30:26.732 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-07-14T11:30:26.732 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.1 2023-07-14T11:30:27.065 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:30:26 smithi045 systemd[1]: Stopping Ceph osd.1 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:27.565 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:30:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:30:27.108+0000 7fe18cbe9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:27.565 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:30:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:30:27.108+0000 7fe18cbe9700 -1 osd.1 119 *** Got signal Terminated *** 2023-07-14T11:30:27.565 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:30:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1[133861]: 2023-07-14T11:30:27.108+0000 7fe18cbe9700 -1 osd.1 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:33.315 INFO:journalctl@ceph.osd.1.smithi045.stdout:Jul 14 11:30:32 smithi045 bash[161225]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-1 2023-07-14T11:30:34.261 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.1.service' 2023-07-14T11:30:34.302 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:34.303 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-07-14T11:30:34.303 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-07-14T11:30:34.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.2 2023-07-14T11:30:34.657 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:30:34 smithi130 systemd[1]: Stopping Ceph osd.2 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:35.043 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:30:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:30:34.653+0000 7f5af0e8d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:35.044 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:30:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:30:34.653+0000 7f5af0e8d700 -1 osd.2 119 *** Got signal Terminated *** 2023-07-14T11:30:35.044 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:30:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2[135221]: 2023-07-14T11:30:34.653+0000 7f5af0e8d700 -1 osd.2 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:40.793 INFO:journalctl@ceph.osd.2.smithi130.stdout:Jul 14 11:30:40 smithi130 bash[162750]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-2 2023-07-14T11:30:41.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.2.service' 2023-07-14T11:30:41.797 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:41.798 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-07-14T11:30:41.798 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-07-14T11:30:41.799 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.3 2023-07-14T11:30:42.297 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:41 smithi040 systemd[1]: Stopping Ceph osd.3 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:42.797 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:42 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:42.332+0000 7f7036736700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:42.797 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:42 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:42.332+0000 7f7036736700 -1 osd.3 119 *** Got signal Terminated *** 2023-07-14T11:30:42.797 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:42 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:42.332+0000 7f7036736700 -1 osd.3 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:44.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:44.213+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:45.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:44 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:44.725+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:45.403 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:45 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:45.132+0000 7f7030527700 -1 osd.3 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.884314+0000 front 2023-07-14T11:30:18.884377+0000 (oldest deadline 2023-07-14T11:30:44.184529+0000) 2023-07-14T11:30:45.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:45.229+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:45.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:45 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:45.400+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:46.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:45 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:45.746+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:46.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:46.218+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:46.546 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:46 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:46.351+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:46.547 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:46 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:46.124+0000 7f7030527700 -1 osd.3 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.884314+0000 front 2023-07-14T11:30:18.884377+0000 (oldest deadline 2023-07-14T11:30:44.184529+0000) 2023-07-14T11:30:47.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:46.685+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:47.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:46 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:46.785+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:47.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:47.198+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:47.547 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:47 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3[115807]: 2023-07-14T11:30:47.086+0000 7f7030527700 -1 osd.3 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.884314+0000 front 2023-07-14T11:30:18.884377+0000 (oldest deadline 2023-07-14T11:30:44.184529+0000) 2023-07-14T11:30:47.547 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:47 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:47.318+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:48.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:47.684+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:48.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:47 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:47.800+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:48.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:48.190+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:48.546 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:48 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:48.284+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:48.547 INFO:journalctl@ceph.osd.3.smithi040.stdout:Jul 14 11:30:48 smithi040 bash[161679]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-3 2023-07-14T11:30:49.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:48.653+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:49.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:48 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:48.837+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:49.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:49 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:49.158+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:49.546 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:49 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:49.274+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:50.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:49 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:49.679+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:50.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:49 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:49.805+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:50.252 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.3.service' 2023-07-14T11:30:50.342 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:50.343 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-07-14T11:30:50.343 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-07-14T11:30:50.344 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.4 2023-07-14T11:30:50.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:50 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:50.111+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:50.547 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:50 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:50.235+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:50.724 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:50 smithi045 systemd[1]: Stopping Ceph osd.4 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:51.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:50 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:50.686+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:51.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:50 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:50.803+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:51.065 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:50 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:30:50.720+0000 7fe2d026d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:51.065 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:50 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:30:50.720+0000 7fe2d026d700 -1 osd.4 119 *** Got signal Terminated *** 2023-07-14T11:30:51.066 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:50 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:30:50.720+0000 7fe2d026d700 -1 osd.4 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:51.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:51 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:51.144+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:51.546 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:51 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:51.274+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:52.045 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:51 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:51.646+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:52.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:51 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:51.774+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:52.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:52 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:52.099+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:52.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:52 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:52.296+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:53.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:52 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:52.616+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:53.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:52 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:52.816+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:53.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:53 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:53.051+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:53.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:53 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:53.323+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:54.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:53 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:53.645+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:54.065 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:53 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:30:53.787+0000 7fe2c905c700 -1 osd.4 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.887119+0000 front 2023-07-14T11:30:29.887073+0000 (oldest deadline 2023-07-14T11:30:53.387271+0000) 2023-07-14T11:30:54.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:53 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:53.744+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:54.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:53 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:53.808+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:54.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:54 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:54.211+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:54.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:54 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:54.052+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:54.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:54 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:54.300+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:54.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:54 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:54.453+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:55.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:54 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:54.613+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:55.044 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:54 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:54.613+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:30:55.065 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:54 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4[139418]: 2023-07-14T11:30:54.775+0000 7fe2c905c700 -1 osd.4 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.887119+0000 front 2023-07-14T11:30:29.887073+0000 (oldest deadline 2023-07-14T11:30:53.387271+0000) 2023-07-14T11:30:55.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:54 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:54.702+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:55.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:54 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:54.812+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:55.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:54 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:54.812+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:30:55.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:55 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:55.171+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:55.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:55 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:55.062+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:55.548 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:55 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:55.435+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:55.548 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:55 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:55.283+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:55.549 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:55 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:55.283+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:30:56.028 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:55 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:55.603+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:56.029 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:55 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:55.603+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:30:56.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:55 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:55.671+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:56.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:55 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:55.826+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:56.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:55 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:55.826+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:30:56.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:56.174+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:56.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:56.025+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:56.491 INFO:journalctl@ceph.osd.4.smithi045.stdout:Jul 14 11:30:56 smithi045 bash[161489]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-4 2023-07-14T11:30:56.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:56 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:56.464+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:56.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:56 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:56.298+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:56.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:56 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:56.298+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:30:56.999 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:56.626+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:56.999 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:56.626+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:30:57.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:56 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:56.661+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:57.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:56 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:56.859+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:57.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:56 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:56.859+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:30:57.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:57 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:57.166+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:57.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:56.995+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:57.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:56 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:56.995+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:30:57.796 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:57 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:57.445+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:57.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:57 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:57.302+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:57.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:57 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:57.302+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:30:57.964 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:57 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:57.698+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:57.965 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:57 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:57.845+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:57.966 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:57 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:57.845+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:30:57.971 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.4.service' 2023-07-14T11:30:57.990 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:57 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:57.615+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:57.990 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:57 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:57.615+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:30:58.013 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:30:58.014 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-07-14T11:30:58.015 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-07-14T11:30:58.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.5 2023-07-14T11:30:58.293 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 systemd[1]: Stopping Ceph osd.5 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:30:58.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:58.129+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:58.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:57 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:57.987+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:58.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:57 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:57.987+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:30:58.793 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:58.375+0000 7ff2e557c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:30:58.794 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:58.375+0000 7ff2e557c700 -1 osd.5 119 *** Got signal Terminated *** 2023-07-14T11:30:58.794 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:58.375+0000 7ff2e557c700 -1 osd.5 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:30:58.794 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:58.618+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:30:58.794 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:58 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:58.618+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:30:58.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:58.431+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:58.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:58.313+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:58.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:58 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:58.313+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:30:59.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:58 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:58.655+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:30:59.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:58 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:58.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:30:59.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:58 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:58.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:30:59.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:30:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:30:59.154+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:30:59.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:59.011+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:30:59.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:30:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:30:59.011+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:30:59.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:30:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:30:59.479+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:30:59.930 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:59.333+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:30:59.930 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:30:59 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:30:59.333+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:00.043 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:59.660+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:31:00.044 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:30:59 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:30:59.660+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:31:00.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:30:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:30:59.636+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:00.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:59.837+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:00.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:30:59 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:30:59.837+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:00.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:00.120+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:00.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:00.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:00.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:00.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:00.796 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:00.432+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:00.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:00.381+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:00.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:00.381+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:00.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:00.381+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:00.888 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:00.629+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:01.036 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:00.613+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:31:01.037 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:00 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:00.613+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:31:01.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:01.097+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:01.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:01.033+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:01.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:01.033+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:01.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:00.885+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:01.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:00.885+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:01.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:00 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:00.885+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:01.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:01.415+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:01.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:01.415+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:01.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:01.389+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:01.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:01.389+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:01.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:01 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:01.389+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:01.923 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:01.613+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:02.009 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:01.654+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:31:02.009 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:01 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:01.654+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:31:02.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:02.063+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:02.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:02.005+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:02.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:02.005+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:02.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:02.005+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:02.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:01.919+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:02.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:01.919+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:02.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:01 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:01.919+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:02.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:02.437+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:02.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:02.437+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:02.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:02.375+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:02.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:02.375+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:02.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:02 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:02.375+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:02.876 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:02.580+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:02.877 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:02.580+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:03.018 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:02.672+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:23.018262+0000 front 2023-07-14T11:30:23.018057+0000 (oldest deadline 2023-07-14T11:30:45.918223+0000) 2023-07-14T11:31:03.019 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:02 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5[140627]: 2023-07-14T11:31:02.672+0000 7ff2deb6c700 -1 osd.5 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:28.719953+0000 front 2023-07-14T11:30:28.719981+0000 (oldest deadline 2023-07-14T11:30:54.520494+0000) 2023-07-14T11:31:03.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:03.054+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:03.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:03.015+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:03.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:03.015+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:03.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:03 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:03.015+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:03.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:02.873+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:03.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:02.873+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:03.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:02 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:02.873+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:03.799 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:03.395+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:03.800 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:03.395+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:03.800 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:03.332+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:03.800 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:03.332+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:03.801 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:03 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:03.332+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:03.888 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:03.607+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:03.889 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:03.607+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:04.295 INFO:journalctl@ceph.osd.5.smithi130.stdout:Jul 14 11:31:04 smithi130 bash[163022]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-5 2023-07-14T11:31:04.296 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:04.044+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:04.297 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:04.059+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:04.298 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:04.059+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:04.298 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:04 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:04.059+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:04.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:03.884+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:04.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:03.884+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:04.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:03 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:03.884+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:04.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:04.415+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:04.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:04.415+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:04.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:04.354+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:04.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:04.354+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:04.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:04.354+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:04.875 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:04.577+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:04.875 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:04.577+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:05.214 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:05.023+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:05.215 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:05.023+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:05.215 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.016+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:05.215 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.016+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:05.216 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.016+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:05.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.5.service' 2023-07-14T11:31:05.260 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:05.261 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-07-14T11:31:05.261 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-07-14T11:31:05.262 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.6 2023-07-14T11:31:05.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:04.871+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:05.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:04.871+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:05.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:04 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:04.871+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:05.782 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:05.401+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:05.782 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:05.401+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:05.783 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:05.401+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:05.783 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 systemd[1]: Stopping Ceph osd.6 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:05.783 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:05.436+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:05.783 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:05.436+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:05.784 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:05.778+0000 7f14a1961700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:05.784 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:05.778+0000 7f14a1961700 -1 osd.6 119 *** Got signal Terminated *** 2023-07-14T11:31:05.839 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:05.583+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:05.840 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:05.583+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:06.047 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:05 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:05.778+0000 7f14a1961700 -1 osd.6 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:06.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:05.998+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:06.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:05.998+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:06.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:06.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:06.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:05 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:05.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:06.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:05.835+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:06.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:05.835+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:06.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:05 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:05.835+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:06.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:06.479+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:06.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:06.479+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:06.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:06.417+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:06.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:06.417+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:06.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:06 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:06.417+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:06.834 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:06.575+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:06.834 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:06.575+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:07.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:07.037+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:07.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:07.037+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:07.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.000+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:07.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.000+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:07.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.000+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:07.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:06.831+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:07.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:06.831+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:07.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:06 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:06.831+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:07.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:07.384+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:07.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:07.384+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:07.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:07.384+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:07.798 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:07.486+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:07.798 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:07 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:07.486+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:07.866 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:07.580+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:07.867 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:07.580+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:08.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:08 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:08.048+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:08.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:08 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:08.048+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:08.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.973+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:08.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.973+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:08.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:07 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:07.973+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:08.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:07.863+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:08.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:07.863+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:08.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:07 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:07.863+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:08.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:08.495+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:08.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:08.495+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:08.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:08.403+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:08.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:08.403+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:08.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:08.403+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:08.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:08 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:08.403+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:08.907 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:08.599+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:08.908 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:08.599+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:09.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:08.999+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:09.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:08.999+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:09.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:08 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:08.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:09.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:08 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:08.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:09.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:08 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:08.986+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:09.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:08.904+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:09.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:08.904+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:09.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:08.904+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:09.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:09.484+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:09.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:09.484+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:09.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:09.406+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:09.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:09.406+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:09.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:09.406+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:09.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:09 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:09.406+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:09.930 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:09.636+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:09.930 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:09.636+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:09.931 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:09.636+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:10.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.016+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:10.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.016+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:10.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.016+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:10.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:09.975+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:10.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:09.975+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:10.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:09 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:09.975+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:10.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:09.927+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:10.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:09.927+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:10.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:09.927+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:10.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:09 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:09.927+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:10.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:10.506+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.040969+0000 front 2023-07-14T11:30:29.041122+0000 (oldest deadline 2023-07-14T11:30:54.340855+0000) 2023-07-14T11:31:10.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6[121359]: 2023-07-14T11:31:10.506+0000 7f149bf53700 -1 osd.6 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.443501+0000 front 2023-07-14T11:30:39.443370+0000 (oldest deadline 2023-07-14T11:31:00.543345+0000) 2023-07-14T11:31:10.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:10.370+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:10.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:10.370+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:10.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:10.370+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:10.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:10 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:10.370+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:10.941 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:10.603+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:10.942 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:10.603+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:10.943 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:10.603+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:11.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.993+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:11.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.993+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:11.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:10.993+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:11.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:10.939+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:11.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:10.939+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:11.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:10 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:10.939+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:11.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:10.938+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:11.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:10.938+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:11.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:10.938+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:11.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:10 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:10.938+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:11.797 INFO:journalctl@ceph.osd.6.smithi040.stdout:Jul 14 11:31:11 smithi040 bash[161950]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-6 2023-07-14T11:31:11.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:11.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:11.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:11.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:11.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:11.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:11.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:11 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:11.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:11.895 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:11.630+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:11.895 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:11.630+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:11.895 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:11.630+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:12.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:12.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:12.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:12.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:11.961+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:12.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:11.961+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:12.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:11 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:11.961+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:12.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:11.892+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:12.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:11.892+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:12.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:11.892+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:12.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:11.892+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:12.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:12.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:12.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:12.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:12.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:12.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:12.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:12 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:12.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:13.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:12.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:13.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:12.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:13.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:12.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:13.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:12.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:13.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:12.624+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:13.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:12.624+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:13.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:12 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:12.624+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:13.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.984+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:13.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.984+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:13.295 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:12.984+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:13.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:12.970+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:13.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:12.970+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:13.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:12 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:12.970+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:13.367 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.6.service' 2023-07-14T11:31:13.461 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:13.462 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-07-14T11:31:13.462 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-07-14T11:31:13.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.7 2023-07-14T11:31:13.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:13 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:13.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:13.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:13 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:13.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:13.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:13 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:13.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:13.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:13 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:13.390+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:13.815 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 systemd[1]: Stopping Ceph osd.7 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:13.815 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.583+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:13.816 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.583+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:13.816 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.583+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:14.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:13.952+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:14.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:13.952+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:14.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:13.952+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:14.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:13.976+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:14.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:13.976+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:14.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:13 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:13.976+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:14.315 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.835+0000 7f7d7f9af700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:14.315 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.835+0000 7f7d7f9af700 -1 osd.7 119 *** Got signal Terminated *** 2023-07-14T11:31:14.316 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:13.835+0000 7f7d7f9af700 -1 osd.7 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:14.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:13.851+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:14.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:13.851+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:14.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:13.851+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:14.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:13 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:13.851+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:14.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:14.407+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:14.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:14.407+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:14.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:14.407+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:14.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:14 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:14.407+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:15.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:14.631+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:15.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:14.631+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:15.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:14.631+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:15.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:14.804+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:15.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:14.804+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:15.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:14.804+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:15.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:14 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:14.804+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:15.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:14 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:14.988+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:15.295 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:14 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:14.988+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:15.295 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:14 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:14.988+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:15.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:15.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:15.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:15.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:15.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:15 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:15.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:15.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:15 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:15.408+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:15.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:15 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:15.408+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:15.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:15 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:15.408+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:15.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:15 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:15.408+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:16.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:15.591+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:16.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:15.591+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:16.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:15.591+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:16.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:15.840+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:16.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:15.840+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:16.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:15.840+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:16.068 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:15.840+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:16.293 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:16.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:16.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:16.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:16.294 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:16.021+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:16.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:16.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:16.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:16.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:16.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:16 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:16.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:16.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:16.383+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:16.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:16.383+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:16.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:16.383+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:16.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:16 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:16.383+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:17.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:16.611+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:17.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:16.611+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:17.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:16.611+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:17.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:16.811+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:17.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:16.811+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:17.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:16.811+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:17.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:16 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:16.811+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:17.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:17.054+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:17.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:17.054+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:17.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:17.054+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:17.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:17.107+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:17.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:17.107+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:17.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:17 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:17.107+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:17.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:17.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:17.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:17.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:17.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:17.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:17.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:17.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:17.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:17 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:17.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:18.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:17.650+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:18.065 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:17.650+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:18.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:17.650+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:18.066 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:17.650+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.297929+0000 front 2023-07-14T11:30:55.297985+0000 (oldest deadline 2023-07-14T11:31:17.598049+0000) 2023-07-14T11:31:18.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:17.766+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:18.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:17.766+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:18.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:17.766+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:18.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:17 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:17.766+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:18.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:18.069+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:18.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:18.069+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:18.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:18.069+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:18.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:18.069+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:18.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:18.134+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:18.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:18.134+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:18.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:18.134+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:18.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:18 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:18.134+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:18.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:18.382+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:18.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:18.382+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:18.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:18.382+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:18.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:18.382+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:18.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:18 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:18.382+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:19.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:18.772+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:19.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:18.772+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:19.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:18.772+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:19.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:18.772+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:19.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:18.640+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:30.194340+0000 front 2023-07-14T11:30:30.194373+0000 (oldest deadline 2023-07-14T11:30:53.694524+0000) 2023-07-14T11:31:19.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:18.640+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.195435+0000 front 2023-07-14T11:30:37.195469+0000 (oldest deadline 2023-07-14T11:31:01.895665+0000) 2023-07-14T11:31:19.067 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:18.640+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.196635+0000 front 2023-07-14T11:30:47.196848+0000 (oldest deadline 2023-07-14T11:31:08.896491+0000) 2023-07-14T11:31:19.068 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:18 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7[144947]: 2023-07-14T11:31:18.640+0000 7f7d7bfa5700 -1 osd.7 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.297929+0000 front 2023-07-14T11:30:55.297985+0000 (oldest deadline 2023-07-14T11:31:17.598049+0000) 2023-07-14T11:31:19.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:19.042+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:19.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:19.042+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:19.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:19.042+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:19.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:19.042+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:19.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:19.094+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:19.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:19.094+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:19.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:19.094+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:19.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:19 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:19.094+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:19.794 INFO:journalctl@ceph.osd.7.smithi045.stdout:Jul 14 11:31:19 smithi045 bash[161758]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-7 2023-07-14T11:31:19.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:19.393+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:19.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:19.393+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:19.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:19.393+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:19.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:19.393+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:19.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:19 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:19.393+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:20.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:19.790+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:20.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:19.791+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:20.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:19.791+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:20.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:19.791+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:20.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:19.791+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:20.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:20.055+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:20.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:20.055+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:20.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:20.055+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:20.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:20.055+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:20.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:20.077+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:20.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:20.077+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:20.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:20.077+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:20.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:20 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:20.077+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:20.796 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:20.431+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:20.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:20.431+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:20.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:20.431+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:20.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:20.431+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:20.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:20 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:20.431+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:21.015 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:20.819+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:21.015 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:20.819+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:21.015 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:20.819+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:21.015 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:20.819+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:21.016 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:20 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:20.819+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:21.022 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.7.service' 2023-07-14T11:31:21.063 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:21.064 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-07-14T11:31:21.064 INFO:tasks.cephadm.osd.8:Stopping osd.8... 2023-07-14T11:31:21.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.8 2023-07-14T11:31:21.417 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.076+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:21.417 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.076+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:21.417 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.076+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:21.418 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.076+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:21.418 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 systemd[1]: Stopping Ceph osd.8 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:21.418 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.413+0000 7f51a316f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.8 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:21.418 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:21.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:21.419 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:21.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:21.419 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:21.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:21.419 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:21.056+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:21.793 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.413+0000 7f51a316f700 -1 osd.8 119 *** Got signal Terminated *** 2023-07-14T11:31:21.794 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:21 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:21.413+0000 7f51a316f700 -1 osd.8 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:21.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:21.420+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:21.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:21.420+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:21.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:21.420+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:21.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:21.420+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:21.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:21 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:21.420+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:22.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:21.821+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:22.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:21.821+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:22.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:21.821+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:22.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:21.821+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:22.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:21 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:21.821+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:22.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:22.082+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:22.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:22.082+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:22.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:22.082+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:22.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:22.082+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:22.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:22.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:22.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:22.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:22.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:22.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:22.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:22 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:22.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:22.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:22.463+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:22.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:22.463+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:22.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:22.463+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:22.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:22.463+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:22.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:22 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:22.463+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:23.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:22.865+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:23.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:22.865+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:23.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:22.865+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:23.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:22.865+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:23.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:22 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:22.865+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:23.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:23.045+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:23.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:23.045+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:23.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:23.045+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:23.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:23.045+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:23.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:23.084+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:23.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:23.084+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:23.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:23.084+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:23.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:23 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:23.084+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:23.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:23.471+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:23.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:23.471+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:23.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:23.471+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:23.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:23.471+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:23.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:23 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:23.471+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:24.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:23.895+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:24.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:23.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:24.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:23.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:24.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:23.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:24.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:23 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:23.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:24.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:24.077+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:24.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:24.077+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:24.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:24.077+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:24.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:24.077+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:24.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:24.101+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:24.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:24.101+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:24.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:24.101+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:24.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:24.101+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:24.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:24.440+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:24.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:24.440+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:24.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:24.440+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:24.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:24.440+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:24.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:24 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:24.440+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:25.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:25.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:25.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:25.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:25.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:25.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:24 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:24.875+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:25.543 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:25.094+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:25.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:25.094+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:25.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:25.094+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:25.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:25.094+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:25.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:25.094+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:30:58.471572+0000 front 2023-07-14T11:30:58.471615+0000 (oldest deadline 2023-07-14T11:31:24.872164+0000) 2023-07-14T11:31:25.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:25.071+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:25.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:25.071+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:25.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:25.071+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:25.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:25.071+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:25.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:25 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:25.071+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:25.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:25.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:25.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:25.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:25.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:25.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:25.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:25.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:25.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:25 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:25.415+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:26.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:26.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:26.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:26.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:26.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:26.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:25 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:25.878+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:26.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:26.134+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.667705+0000 front 2023-07-14T11:30:31.667732+0000 (oldest deadline 2023-07-14T11:30:53.967687+0000) 2023-07-14T11:31:26.544 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:26.134+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:39.268486+0000 front 2023-07-14T11:30:39.268460+0000 (oldest deadline 2023-07-14T11:31:04.469241+0000) 2023-07-14T11:31:26.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:26.134+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.469749+0000 front 2023-07-14T11:30:44.469881+0000 (oldest deadline 2023-07-14T11:31:09.169866+0000) 2023-07-14T11:31:26.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:26.134+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:52.670813+0000 front 2023-07-14T11:30:52.670836+0000 (oldest deadline 2023-07-14T11:31:17.370802+0000) 2023-07-14T11:31:26.545 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8[146089]: 2023-07-14T11:31:26.134+0000 7f519df62700 -1 osd.8 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:30:58.471572+0000 front 2023-07-14T11:30:58.471615+0000 (oldest deadline 2023-07-14T11:31:24.872164+0000) 2023-07-14T11:31:26.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:26.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:26.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:26.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:26.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:26.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:26.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:26.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:26.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:26 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:26.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:26.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:26 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:26.366+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:26.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:26 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:26.366+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:26.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:26 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:26.366+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:26.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:26 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:26.366+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:26.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:26 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:26.366+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:27.044 INFO:journalctl@ceph.osd.8.smithi130.stdout:Jul 14 11:31:27 smithi130 bash[163297]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-8 2023-07-14T11:31:27.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:27.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:27.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:27.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:27.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:27.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:26 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:26.846+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:27.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:27.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:27.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:27.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:27.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:27.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:27.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:27.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:27.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:27.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:27.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:27 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:27.345+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:27.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:27 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:27.345+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:27.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:27 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:27.345+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:27.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:27 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:27.345+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:27.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:27 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:27.345+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:28.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:28 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:28.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:28.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:28 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:28.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:28.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:28 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:28.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:28.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:28 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:28.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:28.296 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:28 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:28.082+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:28.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:28.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:28.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:28.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:28.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:28.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:27 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:27.872+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:28.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.8.service' 2023-07-14T11:31:28.412 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:28.412 INFO:tasks.cephadm.osd.8:Stopped osd.8 2023-07-14T11:31:28.413 INFO:tasks.cephadm.osd.9:Stopping osd.9... 2023-07-14T11:31:28.413 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.9 2023-07-14T11:31:28.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:28.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:28.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:28.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:28.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:28.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:28.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 systemd[1]: Stopping Ceph osd.9 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:29.297 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.926+0000 7f378b7bc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.9 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:29.297 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.926+0000 7f378b7bc700 -1 osd.9 119 *** Got signal Terminated *** 2023-07-14T11:31:29.297 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:28 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:28.926+0000 7f378b7bc700 -1 osd.9 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:29.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:29.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:29.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:29.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:29.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:29.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:28 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:28.874+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:29.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:29.046+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:29.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:29.046+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:29.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:29.046+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:29.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:29.046+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:29.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:29 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:29.046+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:29.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:29.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:29.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:29.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:29.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:29.799 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:29 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:29.424+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:30.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:30.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:30.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:30.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:30.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:30.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:29 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:29.912+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:30.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:30 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:30.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:30.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:30 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:30.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:30.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:30 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:30.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:30.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:30 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:30.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:30.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:30 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:30.060+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:30.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:30.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:30.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:30.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:30.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:30.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:30 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:30.409+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:31.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:31.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:31.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:31.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:31.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:31.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:31.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:31.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:31.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:31.014+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:31.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:31.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:31.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:31.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:31.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:31.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:30 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:30.896+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:31.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:31.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:31.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:31.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:31.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:31.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:31 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:31.364+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:32.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:32.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:32.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:32.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:32.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:32.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:31 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:31.898+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:32.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:32.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:32.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:32.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:32.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:32.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:32 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:32.050+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:32.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:32.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:32.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:32.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:32.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:32.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:32 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:32.387+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:33.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:33.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:33.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:33.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:33.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:33.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:33 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:33.017+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:33.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:33.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:33.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:33.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:33.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:33.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:33.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:32 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:32.922+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:33.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:21.103713+0000 front 2023-07-14T11:30:21.103738+0000 (oldest deadline 2023-07-14T11:30:45.104724+0000) 2023-07-14T11:31:33.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.506677+0000 front 2023-07-14T11:30:31.506690+0000 (oldest deadline 2023-07-14T11:30:55.006511+0000) 2023-07-14T11:31:33.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:35.006989+0000 front 2023-07-14T11:30:35.007024+0000 (oldest deadline 2023-07-14T11:31:00.208075+0000) 2023-07-14T11:31:33.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:47.210193+0000 front 2023-07-14T11:30:47.210004+0000 (oldest deadline 2023-07-14T11:31:08.310125+0000) 2023-07-14T11:31:33.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:53.011454+0000 front 2023-07-14T11:30:53.011575+0000 (oldest deadline 2023-07-14T11:31:16.511605+0000) 2023-07-14T11:31:33.798 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:33 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9[126835]: 2023-07-14T11:31:33.398+0000 7f3783daa700 -1 osd.9 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.814160+0000 front 2023-07-14T11:31:02.814083+0000 (oldest deadline 2023-07-14T11:31:27.513819+0000) 2023-07-14T11:31:34.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:34.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:34.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:34.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:34.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:34.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:34 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:34.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:34.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:34.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:34.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:34.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:34.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:34.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:34.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:33 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:33.889+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:34.797 INFO:journalctl@ceph.osd.9.smithi040.stdout:Jul 14 11:31:34 smithi040 bash[162225]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-9 2023-07-14T11:31:35.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:35.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:35.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:35.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:35.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:35.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:35.022+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:35.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:35.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:35.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:35.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:35.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:35.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:35.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:34 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:34.860+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:36.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:36.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:36.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:36.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:36.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:36.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.019+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:36.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:36.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:36.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:36.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:36.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:36.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:36.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:35 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:35.864+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:36.425 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.9.service' 2023-07-14T11:31:36.515 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:36.515 INFO:tasks.cephadm.osd.9:Stopped osd.9 2023-07-14T11:31:36.516 INFO:tasks.cephadm.osd.10:Stopping osd.10... 2023-07-14T11:31:36.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.10 2023-07-14T11:31:36.815 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 systemd[1]: Stopping Ceph osd.10 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:37.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:37.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:37.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:37.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:37.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:37.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:36 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:36.972+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:37.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:37.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:37.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:37.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:37.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:37.318 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:37.318 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.842+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:37.318 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.880+0000 7f16159b3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.10 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:37.319 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.880+0000 7f16159b3700 -1 osd.10 119 *** Got signal Terminated *** 2023-07-14T11:31:37.319 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:36 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:36.880+0000 7f16159b3700 -1 osd.10 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:38.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:38.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:38.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:38.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:38.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:38.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:38 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:38.006+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:38.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:38.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:38.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:38.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:38.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:38.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:38.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:37 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:37.852+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:39.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:39.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:39.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:39.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:39.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:39.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:39 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:39.009+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:39.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:39.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:39.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:39.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:39.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:39.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:39.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:38 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:38.881+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:40.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:40.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:40.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:40.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:40.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:40.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:40.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:39 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:39.836+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:40.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:40.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:40.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:40.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:40.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:40.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:40 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:40.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:41.315 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:41.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:41.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:41.316 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:41.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:41.317 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:41.318 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:40 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:40.822+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:41.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:41.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:41.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:41.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:41.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:41.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:41 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:41.057+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:42.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:18.614239+0000 front 2023-07-14T11:30:18.614172+0000 (oldest deadline 2023-07-14T11:30:43.913991+0000) 2023-07-14T11:31:42.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:29.115629+0000 front 2023-07-14T11:30:29.115597+0000 (oldest deadline 2023-07-14T11:30:53.815765+0000) 2023-07-14T11:31:42.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:36.716744+0000 front 2023-07-14T11:30:36.716766+0000 (oldest deadline 2023-07-14T11:31:00.816780+0000) 2023-07-14T11:31:42.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6814 osd.3 since back 2023-07-14T11:30:44.218568+0000 front 2023-07-14T11:30:44.218645+0000 (oldest deadline 2023-07-14T11:31:08.918538+0000) 2023-07-14T11:31:42.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:54.219522+0000 front 2023-07-14T11:30:54.219564+0000 (oldest deadline 2023-07-14T11:31:19.519754+0000) 2023-07-14T11:31:42.066 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:03.021090+0000 front 2023-07-14T11:31:03.021134+0000 (oldest deadline 2023-07-14T11:31:24.120796+0000) 2023-07-14T11:31:42.067 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:41 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10[150373]: 2023-07-14T11:31:41.775+0000 7f160efa3700 -1 osd.10 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.622996+0000 front 2023-07-14T11:31:08.623015+0000 (oldest deadline 2023-07-14T11:31:32.122597+0000) 2023-07-14T11:31:42.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:42.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:42.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:42.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:42.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:42.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:42 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:42.054+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:43.065 INFO:journalctl@ceph.osd.10.smithi045.stdout:Jul 14 11:31:42 smithi045 bash[162029]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-10 2023-07-14T11:31:43.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:43.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:43.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:43.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:43.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:43.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:43.547 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:43 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:43.086+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:44.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.10.service' 2023-07-14T11:31:44.087 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:44.088 INFO:tasks.cephadm.osd.10:Stopped osd.10 2023-07-14T11:31:44.089 INFO:tasks.cephadm.osd.11:Stopping osd.11... 2023-07-14T11:31:44.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.11 2023-07-14T11:31:44.446 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:44.446 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:44.446 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:44.447 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:44.447 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:44.447 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:44.447 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.058+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:44.448 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 systemd[1]: Stopping Ceph osd.11 for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:44.793 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.442+0000 7f7e571ac700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.11 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:44.794 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.442+0000 7f7e571ac700 -1 osd.11 119 *** Got signal Terminated *** 2023-07-14T11:31:44.794 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:44 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:44.442+0000 7f7e571ac700 -1 osd.11 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-07-14T11:31:45.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:45.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:45.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:45.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:45.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:45.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:45.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:45 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:45.102+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:46.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:46.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:46.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:46.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:46.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:46.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:46.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:46 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:46.078+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:47.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:47.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:47.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:47.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:47.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:47.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:47.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:47 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:47.072+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:48.543 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:48.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:48.544 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:48.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:48.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:48.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:48.545 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:48.546 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.044+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6820 osd.8 since back 2023-07-14T11:31:22.433616+0000 front 2023-07-14T11:31:22.433577+0000 (oldest deadline 2023-07-14T11:31:47.133602+0000) 2023-07-14T11:31:49.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6806 osd.0 since back 2023-07-14T11:30:22.622164+0000 front 2023-07-14T11:30:22.622064+0000 (oldest deadline 2023-07-14T11:30:43.722088+0000) 2023-07-14T11:31:49.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6804 osd.1 since back 2023-07-14T11:30:31.224933+0000 front 2023-07-14T11:30:31.224813+0000 (oldest deadline 2023-07-14T11:30:56.524981+0000) 2023-07-14T11:31:49.294 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6804 osd.2 since back 2023-07-14T11:30:37.026042+0000 front 2023-07-14T11:30:37.026009+0000 (oldest deadline 2023-07-14T11:31:01.126065+0000) 2023-07-14T11:31:49.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6812 osd.4 since back 2023-07-14T11:30:55.128831+0000 front 2023-07-14T11:30:55.128732+0000 (oldest deadline 2023-07-14T11:31:17.428745+0000) 2023-07-14T11:31:49.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6812 osd.5 since back 2023-07-14T11:31:02.030280+0000 front 2023-07-14T11:31:02.030253+0000 (oldest deadline 2023-07-14T11:31:24.330410+0000) 2023-07-14T11:31:49.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.40:6822 osd.6 since back 2023-07-14T11:31:08.332161+0000 front 2023-07-14T11:31:08.332179+0000 (oldest deadline 2023-07-14T11:31:31.832022+0000) 2023-07-14T11:31:49.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.45:6820 osd.7 since back 2023-07-14T11:31:17.133412+0000 front 2023-07-14T11:31:17.133475+0000 (oldest deadline 2023-07-14T11:31:42.433173+0000) 2023-07-14T11:31:49.295 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:48 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11[151614]: 2023-07-14T11:31:48.997+0000 7f7e537a2700 -1 osd.11 119 heartbeat_check: no reply from 172.21.15.130:6820 osd.8 since back 2023-07-14T11:31:22.433616+0000 front 2023-07-14T11:31:22.433577+0000 (oldest deadline 2023-07-14T11:31:47.133602+0000) 2023-07-14T11:31:50.293 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:50 smithi130 bash[163558]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-osd-11 2023-07-14T11:31:51.306 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:51 smithi130 systemd[1]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.11.service: Succeeded. 2023-07-14T11:31:51.307 INFO:journalctl@ceph.osd.11.smithi130.stdout:Jul 14 11:31:51 smithi130 systemd[1]: Stopped Ceph osd.11 for bbe0b002-2235-11ee-9b32-001a4aab830c. 2023-07-14T11:31:51.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@osd.11.service' 2023-07-14T11:31:51.348 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:51.349 INFO:tasks.cephadm.osd.11:Stopped osd.11 2023-07-14T11:31:51.350 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2023-07-14T11:31:51.350 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.a 2023-07-14T11:31:51.797 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:31:51 smithi040 systemd[1]: Stopping Ceph mds.a for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:52.297 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:31:51 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: 2023-07-14T11:31:51.877+0000 7fa6d9057700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:52.297 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:31:51 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a[132184]: 2023-07-14T11:31:51.877+0000 7fa6d9057700 -1 mds.a *** got signal Terminated *** 2023-07-14T11:31:55.530 INFO:journalctl@ceph.mds.a.smithi040.stdout:Jul 14 11:31:55 smithi040 bash[162504]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-a 2023-07-14T11:31:55.567 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.a.service' 2023-07-14T11:31:55.654 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:55.655 INFO:tasks.cephadm.mds.a:Stopped mds.a 2023-07-14T11:31:55.655 INFO:tasks.cephadm.mds.d:Stopping mds.d... 2023-07-14T11:31:55.656 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.d 2023-07-14T11:31:56.047 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:31:55 smithi040 systemd[1]: Stopping Ceph mds.d for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:31:56.547 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:31:56 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: 2023-07-14T11:31:56.172+0000 7f98888a6700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.d -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:31:56.547 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:31:56 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d[131178]: 2023-07-14T11:31:56.172+0000 7f98888a6700 -1 mds.d *** got signal Terminated *** 2023-07-14T11:31:59.547 INFO:journalctl@ceph.mds.d.smithi040.stdout:Jul 14 11:31:59 smithi040 bash[162642]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-d 2023-07-14T11:31:59.655 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.d.service' 2023-07-14T11:31:59.740 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:31:59.740 INFO:tasks.cephadm.mds.d:Stopped mds.d 2023-07-14T11:31:59.741 INFO:tasks.cephadm.mds.g:Stopping mds.g... 2023-07-14T11:31:59.741 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.g 2023-07-14T11:32:00.047 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:31:59 smithi040 systemd[1]: Stopping Ceph mds.g for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:00.547 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:32:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: 2023-07-14T11:32:00.245+0000 7f89047db700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.g -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:00.548 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:32:00 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g[131793]: 2023-07-14T11:32:00.245+0000 7f89047db700 -1 mds.g *** got signal Terminated *** 2023-07-14T11:32:03.296 INFO:journalctl@ceph.mds.g.smithi040.stdout:Jul 14 11:32:03 smithi040 bash[162787]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-g 2023-07-14T11:32:03.550 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.g.service' 2023-07-14T11:32:03.634 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:03.635 INFO:tasks.cephadm.mds.g:Stopped mds.g 2023-07-14T11:32:03.635 INFO:tasks.cephadm.mds.j:Stopping mds.j... 2023-07-14T11:32:03.635 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.j 2023-07-14T11:32:03.797 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:32:03 smithi040 systemd[1]: Stopping Ceph mds.j for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:04.547 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:32:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-j[130789]: 2023-07-14T11:32:04.127+0000 7fe918527700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.j -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:04.548 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:32:04 smithi040 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-j[130789]: 2023-07-14T11:32:04.127+0000 7fe918527700 -1 mds.j *** got signal Terminated *** 2023-07-14T11:32:07.796 INFO:journalctl@ceph.mds.j.smithi040.stdout:Jul 14 11:32:07 smithi040 bash[162931]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-j 2023-07-14T11:32:07.906 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.j.service' 2023-07-14T11:32:07.991 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:07.991 INFO:tasks.cephadm.mds.j:Stopped mds.j 2023-07-14T11:32:07.992 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2023-07-14T11:32:07.992 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.b 2023-07-14T11:32:08.315 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:32:08 smithi045 systemd[1]: Stopping Ceph mds.b for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:08.815 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:32:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:32:08.409+0000 7f7fe3ee2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:08.815 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:32:08 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b[154157]: 2023-07-14T11:32:08.409+0000 7f7fe3ee2700 -1 mds.b *** got signal Terminated *** 2023-07-14T11:32:11.065 INFO:journalctl@ceph.mds.b.smithi045.stdout:Jul 14 11:32:11 smithi045 bash[162303]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-b 2023-07-14T11:32:11.221 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.b.service' 2023-07-14T11:32:11.260 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:11.260 INFO:tasks.cephadm.mds.b:Stopped mds.b 2023-07-14T11:32:11.261 INFO:tasks.cephadm.mds.e:Stopping mds.e... 2023-07-14T11:32:11.261 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.e 2023-07-14T11:32:11.565 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:32:11 smithi045 systemd[1]: Stopping Ceph mds.e for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:12.315 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:32:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: 2023-07-14T11:32:11.910+0000 7f128ab38700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.e -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:12.316 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:32:11 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e[153473]: 2023-07-14T11:32:11.910+0000 7f128ab38700 -1 mds.e *** got signal Terminated *** 2023-07-14T11:32:15.288 INFO:journalctl@ceph.mds.e.smithi045.stdout:Jul 14 11:32:14 smithi045 bash[162441]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-e 2023-07-14T11:32:15.294 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.e.service' 2023-07-14T11:32:15.329 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:15.329 INFO:tasks.cephadm.mds.e:Stopped mds.e 2023-07-14T11:32:15.330 INFO:tasks.cephadm.mds.h:Stopping mds.h... 2023-07-14T11:32:15.330 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.h 2023-07-14T11:32:15.565 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:32:15 smithi045 systemd[1]: Stopping Ceph mds.h for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:16.065 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:32:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: 2023-07-14T11:32:15.717+0000 7f210a829700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.h -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:16.065 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:32:15 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h[153814]: 2023-07-14T11:32:15.717+0000 7f210a829700 -1 mds.h *** got signal Terminated *** 2023-07-14T11:32:19.276 INFO:journalctl@ceph.mds.h.smithi045.stdout:Jul 14 11:32:18 smithi045 bash[162586]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-h 2023-07-14T11:32:19.282 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.h.service' 2023-07-14T11:32:19.319 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:19.319 INFO:tasks.cephadm.mds.h:Stopped mds.h 2023-07-14T11:32:19.320 INFO:tasks.cephadm.mds.k:Stopping mds.k... 2023-07-14T11:32:19.320 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.k 2023-07-14T11:32:19.565 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:32:19 smithi045 systemd[1]: Stopping Ceph mds.k for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:20.065 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:32:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-k[153129]: 2023-07-14T11:32:19.698+0000 7f0658387700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.k -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:20.066 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:32:19 smithi045 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-k[153129]: 2023-07-14T11:32:19.698+0000 7f0658387700 -1 mds.k *** got signal Terminated *** 2023-07-14T11:32:24.176 INFO:journalctl@ceph.mds.k.smithi045.stdout:Jul 14 11:32:23 smithi045 bash[162732]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-k 2023-07-14T11:32:24.302 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.k.service' 2023-07-14T11:32:24.338 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:24.339 INFO:tasks.cephadm.mds.k:Stopped mds.k 2023-07-14T11:32:24.340 INFO:tasks.cephadm.mds.c:Stopping mds.c... 2023-07-14T11:32:24.340 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.c 2023-07-14T11:32:24.699 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:32:24 smithi130 systemd[1]: Stopping Ceph mds.c for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:25.043 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:32:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: 2023-07-14T11:32:24.694+0000 7f7212d13700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:25.044 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:32:24 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c[155200]: 2023-07-14T11:32:24.694+0000 7f7212d13700 -1 mds.c *** got signal Terminated *** 2023-07-14T11:32:27.044 INFO:journalctl@ceph.mds.c.smithi130.stdout:Jul 14 11:32:26 smithi130 bash[163834]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-c 2023-07-14T11:32:27.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.c.service' 2023-07-14T11:32:27.295 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:27.296 INFO:tasks.cephadm.mds.c:Stopped mds.c 2023-07-14T11:32:27.297 INFO:tasks.cephadm.mds.f:Stopping mds.f... 2023-07-14T11:32:27.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.f 2023-07-14T11:32:27.543 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:32:27 smithi130 systemd[1]: Stopping Ceph mds.f for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:28.043 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:32:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: 2023-07-14T11:32:27.669+0000 7fefdb31a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.f -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:28.044 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:32:27 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f[154859]: 2023-07-14T11:32:27.669+0000 7fefdb31a700 -1 mds.f *** got signal Terminated *** 2023-07-14T11:32:31.043 INFO:journalctl@ceph.mds.f.smithi130.stdout:Jul 14 11:32:30 smithi130 bash[163973]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-f 2023-07-14T11:32:31.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.f.service' 2023-07-14T11:32:31.167 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:31.167 INFO:tasks.cephadm.mds.f:Stopped mds.f 2023-07-14T11:32:31.168 INFO:tasks.cephadm.mds.i:Stopping mds.i... 2023-07-14T11:32:31.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.i 2023-07-14T11:32:31.543 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:32:31 smithi130 systemd[1]: Stopping Ceph mds.i for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:32.043 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:32:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: 2023-07-14T11:32:31.542+0000 7f3ad4434700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.i -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:32.044 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:32:31 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i[155542]: 2023-07-14T11:32:31.542+0000 7f3ad4434700 -1 mds.i *** got signal Terminated *** 2023-07-14T11:32:35.293 INFO:journalctl@ceph.mds.i.smithi130.stdout:Jul 14 11:32:34 smithi130 bash[164111]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-i 2023-07-14T11:32:35.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.i.service' 2023-07-14T11:32:35.346 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:35.347 INFO:tasks.cephadm.mds.i:Stopped mds.i 2023-07-14T11:32:35.347 INFO:tasks.cephadm.mds.l:Stopping mds.l... 2023-07-14T11:32:35.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.l 2023-07-14T11:32:35.709 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:32:35 smithi130 systemd[1]: Stopping Ceph mds.l for bbe0b002-2235-11ee-9b32-001a4aab830c... 2023-07-14T11:32:36.043 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:32:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: 2023-07-14T11:32:35.705+0000 7fc7462d7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mds -n mds.l -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-07-14T11:32:36.044 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:32:35 smithi130 ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l[154513]: 2023-07-14T11:32:35.705+0000 7fc7462d7700 -1 mds.l *** got signal Terminated *** 2023-07-14T11:32:39.206 INFO:journalctl@ceph.mds.l.smithi130.stdout:Jul 14 11:32:38 smithi130 bash[164256]: ceph-bbe0b002-2235-11ee-9b32-001a4aab830c-mds-l 2023-07-14T11:32:39.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bbe0b002-2235-11ee-9b32-001a4aab830c@mds.l.service' 2023-07-14T11:32:39.243 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-07-14T11:32:39.244 INFO:tasks.cephadm.mds.l:Stopped mds.l 2023-07-14T11:32:39.244 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force --keep-logs 2023-07-14T11:32:43.141 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force --keep-logs 2023-07-14T11:32:45.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force --keep-logs 2023-07-14T11:32:47.071 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:32:47.152 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:32:47.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-07-14T11:32:47.221 INFO:tasks.cephadm:Archiving crash dumps... 2023-07-14T11:32:47.224 DEBUG:teuthology.misc:Transferring archived files from smithi040:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi040/crash 2023-07-14T11:32:47.225 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash -- . 2023-07-14T11:32:47.313 INFO:teuthology.orchestra.run.smithi040.stderr:tar: /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash: Cannot open: No such file or directory 2023-07-14T11:32:47.314 INFO:teuthology.orchestra.run.smithi040.stderr:tar: Error is not recoverable: exiting now 2023-07-14T11:32:47.316 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi045/crash 2023-07-14T11:32:47.318 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash -- . 2023-07-14T11:32:47.364 INFO:teuthology.orchestra.run.smithi045.stderr:tar: /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash: Cannot open: No such file or directory 2023-07-14T11:32:47.365 INFO:teuthology.orchestra.run.smithi045.stderr:tar: Error is not recoverable: exiting now 2023-07-14T11:32:47.367 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi130/crash 2023-07-14T11:32:47.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash -- . 2023-07-14T11:32:47.443 INFO:teuthology.orchestra.run.smithi130.stderr:tar: /var/lib/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/crash: Cannot open: No such file or directory 2023-07-14T11:32:47.443 INFO:teuthology.orchestra.run.smithi130.stderr:tar: Error is not recoverable: exiting now 2023-07-14T11:32:47.444 INFO:tasks.cephadm:Checking cluster log for badness... 2023-07-14T11:32:47.445 DEBUG:teuthology.orchestra.run.smithi040:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'overall HEALTH_' | egrep -v '\(FS_DEGRADED\)' | egrep -v '\(MDS_FAILED\)' | egrep -v '\(MDS_DEGRADED\)' | egrep -v '\(FS_WITH_FAILED_MDS\)' | egrep -v '\(MDS_DAMAGE\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(FS_INLINE_DATA_DEPRECATED\)' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v 'slow metadata IO' | egrep -v SLOW_OPS | egrep -v 'slow request' | head -n 1 2023-07-14T11:32:47.528 INFO:teuthology.orchestra.run.smithi040.stderr:grep: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph.log: No such file or directory 2023-07-14T11:32:47.533 INFO:tasks.cephadm:Compressing logs... 2023-07-14T11:32:47.534 DEBUG:teuthology.orchestra.run.smithi040:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-07-14T11:32:47.539 DEBUG:teuthology.orchestra.run.smithi045:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-07-14T11:32:47.542 DEBUG:teuthology.orchestra.run.smithi130:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-07-14T11:32:47.569 INFO:teuthology.orchestra.run.smithi130.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-07-14T11:32:47.569 INFO:teuthology.orchestra.run.smithi130.stderr:: No such file or directory 2023-07-14T11:32:47.570 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.5.log 2023-07-14T11:32:47.570 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.2.log 2023-07-14T11:32:47.572 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.11.log 2023-07-14T11:32:47.572 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.5.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.z.log 2023-07-14T11:32:47.573 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.8.log 2023-07-14T11:32:47.573 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.11.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.z.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.8.log: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2023-07-14T11:32:47.574 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.f.log 2023-07-14T11:32:47.575 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.c.log 2023-07-14T11:32:47.577 INFO:teuthology.orchestra.run.smithi045.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-07-14T11:32:47.577 INFO:teuthology.orchestra.run.smithi045.stderr:: No such file or directory 2023-07-14T11:32:47.578 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.10.log 2023-07-14T11:32:47.581 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.h.log 2023-07-14T11:32:47.582 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.e.log 2023-07-14T11:32:47.582 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.10.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.4.log 2023-07-14T11:32:47.582 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.h.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.e.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.1.log 2023-07-14T11:32:47.582 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.k.log 2023-07-14T11:32:47.583 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log 2023-07-14T11:32:47.583 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.k.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.7.log 2023-07-14T11:32:47.583 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.b.log 2023-07-14T11:32:47.583 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.b.log 2023-07-14T11:32:47.584 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.y.log 2023-07-14T11:32:47.584 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.b.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.y.log: 94.6% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.k.log.gz 2023-07-14T11:32:47.585 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.f.log: /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.c.log: 92.3% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.z.log.gz 2023-07-14T11:32:47.585 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.i.log 2023-07-14T11:32:47.588 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.i.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.c.log 2023-07-14T11:32:47.588 INFO:teuthology.orchestra.run.smithi045.stderr: 92.3% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.y.log.gz 2023-07-14T11:32:47.592 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log 2023-07-14T11:32:47.611 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log: 91.5% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log.gz 2023-07-14T11:32:47.611 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.l.log 2023-07-14T11:32:47.624 INFO:teuthology.orchestra.run.smithi040.stderr:find: ‘/var/log/rbd-target-api’gzip : No such file or directory-5 --verbose 2023-07-14T11:32:47.625 INFO:teuthology.orchestra.run.smithi040.stderr:-- /var/log/ceph/cephadm.log 2023-07-14T11:32:47.625 INFO:teuthology.orchestra.run.smithi040.stderr:gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.0.log 2023-07-14T11:32:47.626 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.g.log 2023-07-14T11:32:47.627 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.a.log 2023-07-14T11:32:47.628 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.g.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.6.log 2023-07-14T11:32:47.629 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.3.log 2023-07-14T11:32:47.630 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.d.log 2023-07-14T11:32:47.631 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.x.log 2023-07-14T11:32:47.632 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.d.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.9.log 2023-07-14T11:32:47.633 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log 2023-07-14T11:32:47.634 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.9.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.j.log 2023-07-14T11:32:47.636 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.a.log 2023-07-14T11:32:47.637 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.j.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.log 2023-07-14T11:32:47.641 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.a.log: /var/log/ceph/ceph-client.admin.log: 93.9% -- replaced with /var/log/ceph/ceph-client.admin.log.gz 2023-07-14T11:32:47.642 INFO:teuthology.orchestra.run.smithi040.stderr: 94.6% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.j.log.gz 2023-07-14T11:32:47.680 INFO:teuthology.orchestra.run.smithi040.stderr: 93.3% -- replaced with /var/log/ceph/cephadm.log.gz 2023-07-14T11:32:47.813 INFO:teuthology.orchestra.run.smithi045.stderr: 94.1% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.e.log.gz 2023-07-14T11:32:47.820 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.l.log: 94.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.i.log.gz 2023-07-14T11:32:47.841 INFO:teuthology.orchestra.run.smithi130.stderr: 94.5% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.l.log.gz 2023-07-14T11:32:47.842 INFO:teuthology.orchestra.run.smithi045.stderr: 91.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log.gz 2023-07-14T11:32:47.976 INFO:teuthology.orchestra.run.smithi045.stderr: 91.9% -- replaced with /var/log/ceph/cephadm.log.gz 2023-07-14T11:32:48.073 INFO:teuthology.orchestra.run.smithi040.stderr: 91.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-volume.log.gz 2023-07-14T11:32:48.284 INFO:teuthology.orchestra.run.smithi040.stderr: 89.7% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mgr.x.log.gz 2023-07-14T11:32:48.321 INFO:teuthology.orchestra.run.smithi130.stderr: 91.7% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.c.log.gz 2023-07-14T11:32:48.996 INFO:teuthology.orchestra.run.smithi045.stderr: 91.8% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.b.log.gz 2023-07-14T11:32:49.351 INFO:teuthology.orchestra.run.smithi040.stderr: 94.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.d.log.gz 2023-07-14T11:32:50.104 INFO:teuthology.orchestra.run.smithi040.stderr: 94.8% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.g.log.gz 2023-07-14T11:32:50.580 INFO:teuthology.orchestra.run.smithi130.stderr: 92.5% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.8.log.gz 2023-07-14T11:32:52.166 INFO:teuthology.orchestra.run.smithi130.stderr: 92.9% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.11.log.gz 2023-07-14T11:32:52.360 INFO:teuthology.orchestra.run.smithi040.stderr: 90.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mon.a.log.gz 2023-07-14T11:32:52.446 INFO:teuthology.orchestra.run.smithi130.stderr: 96.9% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.c.log.gz 2023-07-14T11:32:55.485 INFO:teuthology.orchestra.run.smithi130.stderr: 92.5% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.2.log.gz 2023-07-14T11:32:55.689 INFO:teuthology.orchestra.run.smithi130.stderr: 92.3% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.5.log.gz 2023-07-14T11:32:55.888 INFO:teuthology.orchestra.run.smithi045.stderr: 92.6% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.10.log.gz 2023-07-14T11:32:56.474 INFO:teuthology.orchestra.run.smithi045.stderr: 92.5% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.7.log.gz 2023-07-14T11:32:57.865 INFO:teuthology.orchestra.run.smithi045.stderr: 92.8% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.1.log.gz 2023-07-14T11:33:01.228 INFO:teuthology.orchestra.run.smithi045.stderr: 92.2% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.4.log.gz 2023-07-14T11:33:05.258 INFO:teuthology.orchestra.run.smithi040.stderr: 92.7% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.0.log.gz 2023-07-14T11:33:05.428 INFO:teuthology.orchestra.run.smithi040.stderr: 92.8% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.6.log.gz 2023-07-14T11:33:06.266 INFO:teuthology.orchestra.run.smithi040.stderr: 92.6% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.3.log.gz 2023-07-14T11:33:12.168 INFO:teuthology.orchestra.run.smithi040.stderr: 92.4% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-osd.9.log.gz 2023-07-14T11:33:29.721 INFO:teuthology.orchestra.run.smithi130.stderr: 97.9% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.f.log.gz 2023-07-14T11:33:29.723 INFO:teuthology.orchestra.run.smithi130.stderr: 2023-07-14T11:33:29.724 INFO:teuthology.orchestra.run.smithi130.stderr:real 0m42.165s 2023-07-14T11:33:29.724 INFO:teuthology.orchestra.run.smithi130.stderr:user 1m8.730s 2023-07-14T11:33:29.724 INFO:teuthology.orchestra.run.smithi130.stderr:sys 0m2.821s 2023-07-14T11:33:42.411 INFO:teuthology.orchestra.run.smithi045.stderr: 97.8% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.h.log.gz 2023-07-14T11:36:16.417 INFO:teuthology.orchestra.run.smithi045.stderr: 97.7% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.b.log.gz 2023-07-14T11:36:16.419 INFO:teuthology.orchestra.run.smithi045.stderr: 2023-07-14T11:36:16.419 INFO:teuthology.orchestra.run.smithi045.stderr:real 3m28.862s 2023-07-14T11:36:16.420 INFO:teuthology.orchestra.run.smithi045.stderr:user 3m52.884s 2023-07-14T11:36:16.420 INFO:teuthology.orchestra.run.smithi045.stderr:sys 0m10.252s 2023-07-14T11:41:29.192 INFO:teuthology.orchestra.run.smithi040.stderr: 97.6% -- replaced with /var/log/ceph/bbe0b002-2235-11ee-9b32-001a4aab830c/ceph-mds.a.log.gz 2023-07-14T11:41:29.194 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-07-14T11:41:29.195 INFO:teuthology.orchestra.run.smithi040.stderr:real 8m41.620s 2023-07-14T11:41:29.195 INFO:teuthology.orchestra.run.smithi040.stderr:user 7m15.239s 2023-07-14T11:41:29.195 INFO:teuthology.orchestra.run.smithi040.stderr:sys 1m10.040s 2023-07-14T11:41:29.196 INFO:tasks.cephadm:Archiving logs... 2023-07-14T11:41:29.197 DEBUG:teuthology.misc:Transferring archived files from smithi040:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi040/log 2023-07-14T11:41:29.199 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /var/log/ceph -- . 2023-07-14T11:41:56.843 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi045/log 2023-07-14T11:41:56.844 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/log/ceph -- . 2023-07-14T11:42:12.347 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/log/ceph to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi130/log 2023-07-14T11:42:12.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/log/ceph -- . 2023-07-14T11:42:17.684 INFO:tasks.cephadm:Removing cluster... 2023-07-14T11:42:17.685 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force 2023-07-14T11:42:19.753 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force 2023-07-14T11:42:21.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bbe0b002-2235-11ee-9b32-001a4aab830c --force 2023-07-14T11:42:21.915 INFO:tasks.cephadm:Removing cephadm ... 2023-07-14T11:42:21.916 DEBUG:teuthology.orchestra.run.smithi040:> rm -rf /home/ubuntu/cephtest/cephadm 2023-07-14T11:42:21.966 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf /home/ubuntu/cephtest/cephadm 2023-07-14T11:42:22.001 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/cephadm 2023-07-14T11:42:22.019 INFO:tasks.cephadm:Teardown complete 2023-07-14T11:42:22.020 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-07-14T11:42:22.032 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-07-14T11:42:22.033 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-07-14T11:42:22.038 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-07-14T11:42:22.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-07-14T11:42:22.207 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-07-14T11:42:22.218 INFO:teuthology.task.clock:Checking final clock skew... 2023-07-14T11:42:22.219 DEBUG:teuthology.orchestra.run.smithi040:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T11:42:22.224 DEBUG:teuthology.orchestra.run.smithi045:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T11:42:22.227 DEBUG:teuthology.orchestra.run.smithi130:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-07-14T11:42:22.243 INFO:teuthology.orchestra.run.smithi130.stderr:bash: ntpq: command not found 2023-07-14T11:42:22.244 INFO:teuthology.orchestra.run.smithi045.stderr:bash: ntpq: command not found 2023-07-14T11:42:22.246 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T11:42:22.247 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2023-07-14T11:42:22.247 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 5 -1062us[-1018us] +/- 89ms 2023-07-14T11:42:22.247 INFO:teuthology.orchestra.run.smithi130.stdout:^* hv02.front.sepia.ceph.com 3 6 377 3 -1652us[-1606us] +/- 85ms 2023-07-14T11:42:22.247 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +3124us[+3124us] +/- 101ms 2023-07-14T11:42:22.247 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T11:42:22.262 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T11:42:22.263 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2023-07-14T11:42:22.263 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 3 -1001us[-1001us] +/- 89ms 2023-07-14T11:42:22.263 INFO:teuthology.orchestra.run.smithi045.stdout:^* hv02.front.sepia.ceph.com 3 6 377 7 -1625us[-1722us] +/- 85ms 2023-07-14T11:42:22.263 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 6 +3177us[+3177us] +/- 101ms 2023-07-14T11:42:22.264 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T11:42:22.264 INFO:teuthology.orchestra.run.smithi040.stderr:bash: ntpq: command not found 2023-07-14T11:42:22.292 INFO:teuthology.orchestra.run.smithi040.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-07-14T11:42:22.293 INFO:teuthology.orchestra.run.smithi040.stdout:=============================================================================== 2023-07-14T11:42:22.293 INFO:teuthology.orchestra.run.smithi040.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 3 -1038us[-1038us] +/- 89ms 2023-07-14T11:42:22.293 INFO:teuthology.orchestra.run.smithi040.stdout:^* hv02.front.sepia.ceph.com 3 6 377 4 -1657us[-1733us] +/- 85ms 2023-07-14T11:42:22.293 INFO:teuthology.orchestra.run.smithi040.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 6 +3129us[+3053us] +/- 101ms 2023-07-14T11:42:22.294 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-07-14T11:42:22.295 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-07-14T11:42:22.309 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-07-14T11:42:22.310 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-07-14T11:42:22.320 DEBUG:teuthology.orchestra.run.smithi040:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-07-14T11:42:22.591 DEBUG:teuthology.orchestra.run.smithi045:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-07-14T11:42:22.753 DEBUG:teuthology.orchestra.run.smithi130:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-07-14T11:42:22.848 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T11:42:22.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:22.927 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T11:42:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:22.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-07-14T11:42:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:22.996 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-07-14T11:42:23.011 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-07-14T11:42:23.022 INFO:teuthology.task.internal:Duration was 3607.940832 seconds 2023-07-14T11:42:23.023 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-07-14T11:42:23.038 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-07-14T11:42:23.039 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-07-14T11:42:23.044 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-07-14T11:42:23.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-07-14T11:42:23.092 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T11:42:23.130 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T11:42:23.184 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-07-14T11:42:23.544 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-07-14T11:42:23.545 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi040.front.sepia.ceph.com 2023-07-14T11:42:23.546 DEBUG:teuthology.orchestra.run.smithi040:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-07-14T11:42:23.612 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi045.front.sepia.ceph.com 2023-07-14T11:42:23.612 DEBUG:teuthology.orchestra.run.smithi045:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-07-14T11:42:23.645 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi130.front.sepia.ceph.com 2023-07-14T11:42:23.645 DEBUG:teuthology.orchestra.run.smithi130:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-07-14T11:42:23.668 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-07-14T11:42:23.669 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-07-14T11:42:23.674 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-07-14T11:42:23.687 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-07-14T11:42:23.796 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-07-14T11:42:23.796 DEBUG:teuthology.orchestra.run.smithi040:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-07-14T11:42:23.802 DEBUG:teuthology.orchestra.run.smithi045:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-07-14T11:42:23.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-07-14T11:42:24.035 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-07-14T11:42:24.052 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-07-14T11:42:24.052 DEBUG:teuthology.orchestra.run.smithi040:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-07-14T11:42:24.124 DEBUG:teuthology.orchestra.run.smithi045:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-07-14T11:42:24.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-07-14T11:42:24.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-07-14T11:42:24.199 DEBUG:teuthology.orchestra.run.smithi040:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.205 DEBUG:teuthology.orchestra.run.smithi045:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.248 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = core 2023-07-14T11:42:24.249 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = core 2023-07-14T11:42:24.293 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = core 2023-07-14T11:42:24.330 DEBUG:teuthology.orchestra.run.smithi040:> test -e /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:24.372 DEBUG:teuthology.orchestra.run.smithi045:> test -e /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:24.390 DEBUG:teuthology.orchestra.run.smithi130:> test -e /home/ubuntu/cephtest/archive/coredump 2023-07-14T11:42:24.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-07-14T11:42:24.408 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-07-14T11:42:24.427 INFO:teuthology.task.internal:Transferring archived files... 2023-07-14T11:42:24.429 DEBUG:teuthology.misc:Transferring archived files from smithi040:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi040 2023-07-14T11:42:24.430 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-07-14T11:42:24.525 DEBUG:teuthology.misc:Transferring archived files from smithi045:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi045 2023-07-14T11:42:24.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-07-14T11:42:24.571 DEBUG:teuthology.misc:Transferring archived files from smithi130:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403/remote/smithi130 2023-07-14T11:42:24.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-07-14T11:42:24.621 INFO:teuthology.task.internal:Removing archive directory... 2023-07-14T11:42:24.622 DEBUG:teuthology.orchestra.run.smithi040:> rm -rf -- /home/ubuntu/cephtest/archive 2023-07-14T11:42:24.629 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf -- /home/ubuntu/cephtest/archive 2023-07-14T11:42:24.632 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf -- /home/ubuntu/cephtest/archive 2023-07-14T11:42:24.666 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-07-14T11:42:24.688 INFO:teuthology.task.internal:Not uploading archives. 2023-07-14T11:42:24.688 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-07-14T11:42:24.716 INFO:teuthology.task.internal:Tidying up after the test... 2023-07-14T11:42:24.716 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-07-14T11:42:24.722 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-07-14T11:42:24.725 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-07-14T11:42:24.740 INFO:teuthology.orchestra.run.smithi045.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 14 11:42 /home/ubuntu/cephtest 2023-07-14T11:42:24.742 INFO:teuthology.orchestra.run.smithi130.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 14 11:42 /home/ubuntu/cephtest 2023-07-14T11:42:24.760 INFO:teuthology.orchestra.run.smithi040.stdout: 524293 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Jul 14 11:42 /home/ubuntu/cephtest 2023-07-14T11:42:24.763 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-07-14T11:42:24.778 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-07-14T11:42:24.806 INFO:teuthology.nuke:Checking targets against current locks 2023-07-14T11:42:24.838 DEBUG:teuthology.nuke:shortname: smithi040 2023-07-14T11:42:24.838 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-07-14T11:42:24.865 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.462832', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac='} 2023-07-14T11:42:24.873 DEBUG:teuthology.nuke:shortname: smithi045 2023-07-14T11:42:24.873 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-07-14T11:42:24.902 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi045.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.466833', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac='} 2023-07-14T11:42:24.903 DEBUG:teuthology.nuke:shortname: smithi130 2023-07-14T11:42:24.904 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-07-14T11:42:24.930 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/rishabh-2023-07-14_10:26:42-fs-wip-rishabh-2023Jul13-testing-default-smithi/7337403', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-07-14 10:27:19.464971', 'locked_by': 'scheduled_rishabh@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpUxo5ay/ucdkX9rleNMm8Ofp24XRXUtJPOK3Ji3XBpTiUgmmwneaY6316+3CG+k4K0p3IebanuyTvZApjm02/dNCvu8yD3rjEpWVgIuqYH8O3lEdN9DXcCX+7xHJE0m18+ZPsJQVsNrNJxY+5zqyNnBc8jpZq2LStsWwPOKPWhauANRkOBFSGDoLppWZbLx/yJs+R0qoNLLK9LpKQTUnqsapXvbD27OpPahzj/qbNxFPPyY6XrrldZ6qEO+EjJOujFLH3gzv0kJu10DCuzxrpz6ston4RPi1O3GgcHnF4Mqr/W9VwZ94UTMh+k4dIFRVqJEKrcwRHa5kFsvH/WhVx0Ygr/MiV/Q86mmV7JT+/MpzIRTtWyei5L9IBeZIE9cwzj+mOk6SD/t1roIVEGVV/uF2AyFDVBPr6F8oZ822OGpUyyaDsWTitPNdTX4TC3zdSrLc9tyDa1P74eH/ZLeTSf+C6pM/cepOhQaa6XzpDiPSuJUm9HOQxUlEAvcqa7Ac='} 2023-07-14T11:42:24.941 INFO:teuthology.orchestra.console:Power off smithi040 2023-07-14T11:42:24.943 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-07-14T11:42:24.967 INFO:teuthology.orchestra.console:Power off smithi045 2023-07-14T11:42:24.968 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi045.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-07-14T11:42:24.986 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:25.002 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi045.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:25.034 INFO:teuthology.orchestra.console:Power off smithi130 2023-07-14T11:42:25.036 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-07-14T11:42:25.067 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:29.020 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:29.051 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi045.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:29.095 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:33.136 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:33.164 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi045.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:33.214 DEBUG:teuthology.orchestra.console:pexpect command: ipmitool -H smithi130.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-07-14T11:42:33.350 INFO:teuthology.orchestra.console:Power off for smithi040 completed 2023-07-14T11:42:33.378 INFO:teuthology.orchestra.console:Power off for smithi045 completed 2023-07-14T11:42:33.430 INFO:teuthology.orchestra.console:Power off for smithi130 completed 2023-07-14T11:42:33.538 INFO:teuthology.run:Summary data: client.0-kernel-sha1: a440c79eee0258ae07e4988d8690fe6187b8b7a3 description: fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{base/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/crc wsync/yes} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{cephsqlite-timeout frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/multi/{export-check n/5 replication/default} standby-replay tasks/{0-subvolume/{no-subvolume} 1-check-counter 2-scrub/yes 3-snaps/yes 4-flush/no 5-workunit/postgres}} duration: 3607.9408316612244 failure_reason: 'Command failed on smithi040 with status 1: "sudo TESTDIR=/home/ubuntu/cephtest bash -c ''sudo -u postgres -- pgbench -s 500 -i''"' flavor: default owner: scheduled_rishabh@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=98a6da5322b64d0d8a354d268a14b0f7 status: fail success: false 2023-07-14T11:42:33.539 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-07-14T11:42:33.641 INFO:teuthology.run:FAIL